2024-12-07 18:13:32,044 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-07 18:13:32,055 main DEBUG Took 0.009443 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-07 18:13:32,056 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-07 18:13:32,056 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-07 18:13:32,057 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-07 18:13:32,058 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,065 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-07 18:13:32,076 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,078 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,078 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,079 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,079 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,080 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,080 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,081 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,082 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,084 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,085 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,085 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,086 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,087 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,088 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,089 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 18:13:32,090 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,090 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-07 18:13:32,092 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 18:13:32,094 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-07 18:13:32,096 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-07 18:13:32,097 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-07 18:13:32,098 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-07 18:13:32,099 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-07 18:13:32,111 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-07 18:13:32,114 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-07 18:13:32,116 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-07 18:13:32,116 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-07 18:13:32,117 main DEBUG createAppenders(={Console}) 2024-12-07 18:13:32,118 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-07 18:13:32,118 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-07 18:13:32,119 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-07 18:13:32,119 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-07 18:13:32,120 main DEBUG OutputStream closed 2024-12-07 18:13:32,120 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-07 18:13:32,120 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-07 18:13:32,121 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-07 18:13:32,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-07 18:13:32,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-07 18:13:32,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-07 18:13:32,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-07 18:13:32,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-07 18:13:32,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-07 18:13:32,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-07 18:13:32,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-07 18:13:32,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-07 18:13:32,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-07 18:13:32,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-07 18:13:32,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-07 18:13:32,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-07 18:13:32,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-07 18:13:32,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-07 18:13:32,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-07 18:13:32,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-07 18:13:32,222 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-07 18:13:32,224 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07 18:13:32,225 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-07 18:13:32,225 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-07 18:13:32,226 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-07T18:13:32,499 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98 2024-12-07 18:13:32,503 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-07 18:13:32,503 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07T18:13:32,513 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-07T18:13:32,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-07T18:13:32,562 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/cluster_452d438f-2bce-cc52-05da-ff1816a8b998, deleteOnExit=true 2024-12-07T18:13:32,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-07T18:13:32,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/test.cache.data in system properties and HBase conf 2024-12-07T18:13:32,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/hadoop.tmp.dir in system properties and HBase conf 2024-12-07T18:13:32,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/hadoop.log.dir in system properties and HBase conf 2024-12-07T18:13:32,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-07T18:13:32,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-07T18:13:32,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-07T18:13:32,670 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-07T18:13:32,758 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T18:13:32,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-07T18:13:32,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-07T18:13:32,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-07T18:13:32,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-07T18:13:32,763 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-07T18:13:32,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-07T18:13:32,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-07T18:13:32,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-07T18:13:32,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-07T18:13:32,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/nfs.dump.dir in system properties and HBase conf 2024-12-07T18:13:32,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/java.io.tmpdir in system properties and HBase conf 2024-12-07T18:13:32,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-07T18:13:32,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-07T18:13:32,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-07T18:13:33,641 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-07T18:13:33,731 INFO [Time-limited test {}] log.Log(170): Logging initialized @2352ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-07T18:13:33,814 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T18:13:33,873 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T18:13:33,894 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T18:13:33,894 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T18:13:33,896 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-07T18:13:33,908 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T18:13:33,911 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/hadoop.log.dir/,AVAILABLE} 2024-12-07T18:13:33,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T18:13:34,120 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/java.io.tmpdir/jetty-localhost-44125-hadoop-hdfs-3_4_1-tests_jar-_-any-17324744202067310895/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-07T18:13:34,128 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:44125} 2024-12-07T18:13:34,128 INFO [Time-limited test {}] server.Server(415): Started @2750ms 2024-12-07T18:13:34,526 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-07T18:13:34,533 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-07T18:13:34,534 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-07T18:13:34,534 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-07T18:13:34,534 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-07T18:13:34,535 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/hadoop.log.dir/,AVAILABLE} 2024-12-07T18:13:34,536 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-07T18:13:34,667 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/java.io.tmpdir/jetty-localhost-40171-hadoop-hdfs-3_4_1-tests_jar-_-any-14343377131678706252/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T18:13:34,668 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40171} 2024-12-07T18:13:34,668 INFO [Time-limited test {}] server.Server(415): Started @3290ms 2024-12-07T18:13:34,723 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-07T18:13:35,277 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/cluster_452d438f-2bce-cc52-05da-ff1816a8b998/data/data2/current/BP-419382850-172.17.0.2-1733595213355/current, will proceed with Du for space computation calculation, 2024-12-07T18:13:35,277 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/cluster_452d438f-2bce-cc52-05da-ff1816a8b998/data/data1/current/BP-419382850-172.17.0.2-1733595213355/current, will proceed with Du for space computation calculation, 2024-12-07T18:13:35,330 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-07T18:13:35,392 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9258b75cccfe85a7 with lease ID 0xc95dd8ce2fbd2fdf: Processing first storage report for DS-f94ff075-8ceb-4f8e-bac7-14d4a68a90fe from datanode DatanodeRegistration(127.0.0.1:37437, datanodeUuid=cbc12a6e-c0e1-4814-8304-16012a73b004, infoPort=42907, infoSecurePort=0, ipcPort=42989, storageInfo=lv=-57;cid=testClusterID;nsid=1175285904;c=1733595213355) 2024-12-07T18:13:35,394 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9258b75cccfe85a7 with lease ID 0xc95dd8ce2fbd2fdf: from storage DS-f94ff075-8ceb-4f8e-bac7-14d4a68a90fe node DatanodeRegistration(127.0.0.1:37437, datanodeUuid=cbc12a6e-c0e1-4814-8304-16012a73b004, infoPort=42907, infoSecurePort=0, ipcPort=42989, storageInfo=lv=-57;cid=testClusterID;nsid=1175285904;c=1733595213355), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-12-07T18:13:35,395 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9258b75cccfe85a7 with lease ID 0xc95dd8ce2fbd2fdf: Processing first storage report for DS-eb1ee689-f9be-4e3f-b3c3-cfe589cdfc9f from datanode DatanodeRegistration(127.0.0.1:37437, datanodeUuid=cbc12a6e-c0e1-4814-8304-16012a73b004, infoPort=42907, infoSecurePort=0, ipcPort=42989, storageInfo=lv=-57;cid=testClusterID;nsid=1175285904;c=1733595213355) 2024-12-07T18:13:35,395 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9258b75cccfe85a7 with lease ID 0xc95dd8ce2fbd2fdf: from storage DS-eb1ee689-f9be-4e3f-b3c3-cfe589cdfc9f node DatanodeRegistration(127.0.0.1:37437, datanodeUuid=cbc12a6e-c0e1-4814-8304-16012a73b004, infoPort=42907, infoSecurePort=0, ipcPort=42989, storageInfo=lv=-57;cid=testClusterID;nsid=1175285904;c=1733595213355), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-07T18:13:35,439 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98 2024-12-07T18:13:35,519 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/cluster_452d438f-2bce-cc52-05da-ff1816a8b998/zookeeper_0, clientPort=62924, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/cluster_452d438f-2bce-cc52-05da-ff1816a8b998/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/cluster_452d438f-2bce-cc52-05da-ff1816a8b998/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T18:13:35,529 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62924 2024-12-07T18:13:35,543 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T18:13:35,547 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T18:13:35,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741825_1001 (size=7) 2024-12-07T18:13:36,177 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68 with version=8 2024-12-07T18:13:36,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/hbase-staging 2024-12-07T18:13:36,277 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-07T18:13:36,542 INFO [Time-limited test {}] client.ConnectionUtils(128): master/92acd7e2389b:0 server-side Connection retries=45 2024-12-07T18:13:36,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:13:36,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T18:13:36,558 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T18:13:36,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:13:36,558 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T18:13:36,693 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T18:13:36,754 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-07T18:13:36,763 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-07T18:13:36,767 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T18:13:36,794 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3060 (auto-detected) 2024-12-07T18:13:36,795 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-07T18:13:36,819 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38531 2024-12-07T18:13:36,845 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38531 connecting to ZooKeeper ensemble=127.0.0.1:62924 2024-12-07T18:13:36,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385310x0, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T18:13:36,887 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38531-0x1018e21a11b0000 connected 2024-12-07T18:13:36,933 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T18:13:36,936 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T18:13:36,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:13:36,953 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68, hbase.cluster.distributed=false 2024-12-07T18:13:36,979 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T18:13:36,984 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38531 2024-12-07T18:13:36,985 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38531 2024-12-07T18:13:36,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38531 2024-12-07T18:13:36,989 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38531 2024-12-07T18:13:36,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38531 2024-12-07T18:13:37,123 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/92acd7e2389b:0 server-side Connection retries=45 2024-12-07T18:13:37,125 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:13:37,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T18:13:37,126 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T18:13:37,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T18:13:37,126 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T18:13:37,129 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T18:13:37,132 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T18:13:37,133 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33947 2024-12-07T18:13:37,136 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33947 connecting to ZooKeeper ensemble=127.0.0.1:62924 2024-12-07T18:13:37,137 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T18:13:37,141 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T18:13:37,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:339470x0, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T18:13:37,169 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:339470x0, quorum=127.0.0.1:62924, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:13:37,173 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33947-0x1018e21a11b0001 connected 2024-12-07T18:13:37,173 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T18:13:37,186 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T18:13:37,190 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T18:13:37,197 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T18:13:37,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33947 2024-12-07T18:13:37,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33947 2024-12-07T18:13:37,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33947 2024-12-07T18:13:37,236 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33947 2024-12-07T18:13:37,237 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33947 2024-12-07T18:13:37,263 DEBUG [M:0;92acd7e2389b:38531 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;92acd7e2389b:38531 2024-12-07T18:13:37,264 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/92acd7e2389b,38531,1733595216342 2024-12-07T18:13:37,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:13:37,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:13:37,277 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/92acd7e2389b,38531,1733595216342 2024-12-07T18:13:37,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T18:13:37,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:37,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:37,318 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T18:13:37,320 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/92acd7e2389b,38531,1733595216342 from backup master directory 2024-12-07T18:13:37,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/92acd7e2389b,38531,1733595216342 2024-12-07T18:13:37,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:13:37,324 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T18:13:37,327 WARN [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T18:13:37,328 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=92acd7e2389b,38531,1733595216342 2024-12-07T18:13:37,330 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-07T18:13:37,336 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-07T18:13:37,428 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/hbase.id] with ID: 035791d7-7975-48fc-9d8e-4ea192b86b0e 2024-12-07T18:13:37,429 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/.tmp/hbase.id 2024-12-07T18:13:37,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741826_1002 (size=42) 2024-12-07T18:13:37,472 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/.tmp/hbase.id]:[hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/hbase.id] 2024-12-07T18:13:37,538 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-07T18:13:37,543 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T18:13:37,570 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-12-07T18:13:37,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:37,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:37,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741827_1003 (size=196) 2024-12-07T18:13:37,612 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:13:37,617 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T18:13:37,644 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:37,650 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:13:37,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741828_1004 (size=1189) 2024-12-07T18:13:38,113 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store 2024-12-07T18:13:38,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741829_1005 (size=34) 2024-12-07T18:13:38,136 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-07T18:13:38,139 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:13:38,140 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T18:13:38,141 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:13:38,141 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:13:38,142 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T18:13:38,143 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:13:38,143 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:13:38,144 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733595218140Disabling compacts and flushes for region at 1733595218140Disabling writes for close at 1733595218142 (+2 ms)Writing region close event to WAL at 1733595218143 (+1 ms)Closed at 1733595218143 2024-12-07T18:13:38,147 WARN [master/92acd7e2389b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/.initializing 2024-12-07T18:13:38,147 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/WALs/92acd7e2389b,38531,1733595216342 2024-12-07T18:13:38,155 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T18:13:38,171 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=92acd7e2389b%2C38531%2C1733595216342, suffix=, logDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/WALs/92acd7e2389b,38531,1733595216342, archiveDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/oldWALs, maxLogs=10 2024-12-07T18:13:38,197 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/WALs/92acd7e2389b,38531,1733595216342/92acd7e2389b%2C38531%2C1733595216342.1733595218177, exclude list is [], retry=0 2024-12-07T18:13:38,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37437,DS-f94ff075-8ceb-4f8e-bac7-14d4a68a90fe,DISK] 2024-12-07T18:13:38,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-07T18:13:38,256 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/WALs/92acd7e2389b,38531,1733595216342/92acd7e2389b%2C38531%2C1733595216342.1733595218177 2024-12-07T18:13:38,257 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42907:42907)] 2024-12-07T18:13:38,258 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:13:38,258 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:13:38,262 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,263 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,329 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T18:13:38,334 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:38,337 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:13:38,338 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,344 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T18:13:38,344 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:38,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:13:38,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T18:13:38,349 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:38,351 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:13:38,351 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,354 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T18:13:38,354 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:38,355 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:13:38,356 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,361 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,362 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,367 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,368 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,372 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T18:13:38,376 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T18:13:38,381 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:13:38,383 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73783503, jitterRate=0.09945987164974213}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T18:13:38,389 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733595218275Initializing all the Stores at 1733595218277 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733595218278 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595218279 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595218279Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595218279Cleaning up temporary data from old regions at 1733595218369 (+90 ms)Region opened successfully at 1733595218388 (+19 ms) 2024-12-07T18:13:38,390 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T18:13:38,428 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a48a42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=92acd7e2389b/172.17.0.2:0 2024-12-07T18:13:38,464 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T18:13:38,481 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T18:13:38,481 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T18:13:38,485 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T18:13:38,487 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-07T18:13:38,495 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-07T18:13:38,495 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T18:13:38,539 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T18:13:38,550 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T18:13:38,552 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T18:13:38,555 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T18:13:38,556 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T18:13:38,559 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T18:13:38,561 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T18:13:38,565 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T18:13:38,570 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T18:13:38,571 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T18:13:38,573 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T18:13:38,591 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T18:13:38,593 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T18:13:38,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T18:13:38,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T18:13:38,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:38,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:38,602 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=92acd7e2389b,38531,1733595216342, sessionid=0x1018e21a11b0000, setting cluster-up flag (Was=false) 2024-12-07T18:13:38,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:38,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:38,622 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T18:13:38,624 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=92acd7e2389b,38531,1733595216342 2024-12-07T18:13:38,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:38,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:38,637 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T18:13:38,638 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=92acd7e2389b,38531,1733595216342 2024-12-07T18:13:38,644 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T18:13:38,718 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T18:13:38,728 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T18:13:38,735 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T18:13:38,740 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 92acd7e2389b,38531,1733595216342 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T18:13:38,745 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(746): ClusterId : 035791d7-7975-48fc-9d8e-4ea192b86b0e 2024-12-07T18:13:38,747 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T18:13:38,747 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/92acd7e2389b:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:13:38,748 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/92acd7e2389b:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:13:38,748 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/92acd7e2389b:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:13:38,748 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/92acd7e2389b:0, corePoolSize=5, maxPoolSize=5 2024-12-07T18:13:38,748 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/92acd7e2389b:0, corePoolSize=10, maxPoolSize=10 2024-12-07T18:13:38,748 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,749 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/92acd7e2389b:0, corePoolSize=2, maxPoolSize=2 2024-12-07T18:13:38,749 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,753 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733595248753 2024-12-07T18:13:38,753 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T18:13:38,753 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T18:13:38,754 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T18:13:38,755 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T18:13:38,755 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T18:13:38,756 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T18:13:38,759 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T18:13:38,760 DEBUG [RS:0;92acd7e2389b:33947 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@387a3fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=92acd7e2389b/172.17.0.2:0 2024-12-07T18:13:38,761 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T18:13:38,761 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:38,761 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T18:13:38,761 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T18:13:38,761 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T18:13:38,762 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T18:13:38,763 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,769 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T18:13:38,770 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T18:13:38,770 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T18:13:38,773 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T18:13:38,773 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T18:13:38,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741831_1007 (size=1321) 2024-12-07T18:13:38,775 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/92acd7e2389b:0:becomeActiveMaster-HFileCleaner.large.0-1733595218775,5,FailOnTimeoutGroup] 2024-12-07T18:13:38,776 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/92acd7e2389b:0:becomeActiveMaster-HFileCleaner.small.0-1733595218775,5,FailOnTimeoutGroup] 2024-12-07T18:13:38,776 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,776 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T18:13:38,777 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T18:13:38,777 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68 2024-12-07T18:13:38,778 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,778 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;92acd7e2389b:33947 2024-12-07T18:13:38,779 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,783 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T18:13:38,783 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T18:13:38,783 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T18:13:38,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741832_1008 (size=32) 2024-12-07T18:13:38,786 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(2659): reportForDuty to master=92acd7e2389b,38531,1733595216342 with port=33947, startcode=1733595217075 2024-12-07T18:13:38,797 DEBUG [RS:0;92acd7e2389b:33947 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T18:13:38,833 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33239, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T18:13:38,840 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38531 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 92acd7e2389b,33947,1733595217075 2024-12-07T18:13:38,842 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38531 {}] master.ServerManager(517): Registering regionserver=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:38,857 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68 2024-12-07T18:13:38,857 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35585 2024-12-07T18:13:38,857 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T18:13:38,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T18:13:38,862 DEBUG [RS:0;92acd7e2389b:33947 {}] zookeeper.ZKUtil(111): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/92acd7e2389b,33947,1733595217075 2024-12-07T18:13:38,862 WARN [RS:0;92acd7e2389b:33947 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T18:13:38,862 INFO [RS:0;92acd7e2389b:33947 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:13:38,862 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/WALs/92acd7e2389b,33947,1733595217075 2024-12-07T18:13:38,864 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [92acd7e2389b,33947,1733595217075] 2024-12-07T18:13:38,887 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T18:13:38,900 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T18:13:38,905 INFO [RS:0;92acd7e2389b:33947 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T18:13:38,905 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,906 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T18:13:38,911 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T18:13:38,913 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,913 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,913 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,913 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,913 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,914 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,914 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/92acd7e2389b:0, corePoolSize=2, maxPoolSize=2 2024-12-07T18:13:38,914 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,914 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,914 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,914 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,914 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,915 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/92acd7e2389b:0, corePoolSize=1, maxPoolSize=1 2024-12-07T18:13:38,915 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/92acd7e2389b:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:13:38,915 DEBUG [RS:0;92acd7e2389b:33947 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0, corePoolSize=3, maxPoolSize=3 2024-12-07T18:13:38,916 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,916 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,916 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,916 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,916 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,916 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=92acd7e2389b,33947,1733595217075-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T18:13:38,936 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T18:13:38,938 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=92acd7e2389b,33947,1733595217075-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,938 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,939 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.Replication(171): 92acd7e2389b,33947,1733595217075 started 2024-12-07T18:13:38,958 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:38,958 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1482): Serving as 92acd7e2389b,33947,1733595217075, RpcServer on 92acd7e2389b/172.17.0.2:33947, sessionid=0x1018e21a11b0001 2024-12-07T18:13:38,959 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T18:13:38,959 DEBUG [RS:0;92acd7e2389b:33947 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 92acd7e2389b,33947,1733595217075 2024-12-07T18:13:38,959 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '92acd7e2389b,33947,1733595217075' 2024-12-07T18:13:38,960 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T18:13:38,961 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T18:13:38,961 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T18:13:38,962 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T18:13:38,962 DEBUG [RS:0;92acd7e2389b:33947 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 92acd7e2389b,33947,1733595217075 2024-12-07T18:13:38,962 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '92acd7e2389b,33947,1733595217075' 2024-12-07T18:13:38,962 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T18:13:38,962 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T18:13:38,963 DEBUG [RS:0;92acd7e2389b:33947 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T18:13:38,963 INFO [RS:0;92acd7e2389b:33947 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T18:13:38,963 INFO [RS:0;92acd7e2389b:33947 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T18:13:39,069 INFO [RS:0;92acd7e2389b:33947 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T18:13:39,072 INFO [RS:0;92acd7e2389b:33947 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=92acd7e2389b%2C33947%2C1733595217075, suffix=, logDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/WALs/92acd7e2389b,33947,1733595217075, archiveDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/oldWALs, maxLogs=32 2024-12-07T18:13:39,090 DEBUG [RS:0;92acd7e2389b:33947 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/WALs/92acd7e2389b,33947,1733595217075/92acd7e2389b%2C33947%2C1733595217075.1733595219075, exclude list is [], retry=0 2024-12-07T18:13:39,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37437,DS-f94ff075-8ceb-4f8e-bac7-14d4a68a90fe,DISK] 2024-12-07T18:13:39,098 INFO [RS:0;92acd7e2389b:33947 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/WALs/92acd7e2389b,33947,1733595217075/92acd7e2389b%2C33947%2C1733595217075.1733595219075 2024-12-07T18:13:39,098 DEBUG [RS:0;92acd7e2389b:33947 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42907:42907)] 2024-12-07T18:13:39,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:13:39,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T18:13:39,193 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T18:13:39,194 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:39,194 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:13:39,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T18:13:39,197 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T18:13:39,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:39,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:13:39,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T18:13:39,200 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T18:13:39,200 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:39,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:13:39,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T18:13:39,203 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T18:13:39,203 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:39,204 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:13:39,204 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T18:13:39,205 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740 2024-12-07T18:13:39,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740 2024-12-07T18:13:39,209 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T18:13:39,209 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T18:13:39,210 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:13:39,212 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T18:13:39,216 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:13:39,216 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74570465, jitterRate=0.11118651926517487}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:13:39,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733595219189Initializing all the Stores at 1733595219190 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733595219190Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733595219191 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595219191Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733595219191Cleaning up temporary data from old regions at 1733595219209 (+18 ms)Region opened successfully at 1733595219219 (+10 ms) 2024-12-07T18:13:39,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T18:13:39,219 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T18:13:39,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T18:13:39,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T18:13:39,219 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T18:13:39,221 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T18:13:39,221 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733595219219Disabling compacts and flushes for region at 1733595219219Disabling writes for close at 1733595219219Writing region close event to WAL at 1733595219220 (+1 ms)Closed at 1733595219221 (+1 ms) 2024-12-07T18:13:39,225 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T18:13:39,225 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T18:13:39,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T18:13:39,240 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T18:13:39,242 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T18:13:39,395 DEBUG [92acd7e2389b:38531 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-07T18:13:39,407 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:39,415 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 92acd7e2389b,33947,1733595217075, state=OPENING 2024-12-07T18:13:39,422 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T18:13:39,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:39,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:13:39,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:13:39,426 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:13:39,427 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T18:13:39,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:13:39,609 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T18:13:39,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39707, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T18:13:39,622 INFO [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T18:13:39,622 INFO [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T18:13:39,623 INFO [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T18:13:39,626 INFO [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=92acd7e2389b%2C33947%2C1733595217075.meta, suffix=.meta, logDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/WALs/92acd7e2389b,33947,1733595217075, archiveDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/oldWALs, maxLogs=32 2024-12-07T18:13:39,642 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/WALs/92acd7e2389b,33947,1733595217075/92acd7e2389b%2C33947%2C1733595217075.meta.1733595219627.meta, exclude list is [], retry=0 2024-12-07T18:13:39,645 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37437,DS-f94ff075-8ceb-4f8e-bac7-14d4a68a90fe,DISK] 2024-12-07T18:13:39,648 INFO [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/WALs/92acd7e2389b,33947,1733595217075/92acd7e2389b%2C33947%2C1733595217075.meta.1733595219627.meta 2024-12-07T18:13:39,649 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42907:42907)] 2024-12-07T18:13:39,649 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:13:39,651 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T18:13:39,654 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T18:13:39,658 INFO [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T18:13:39,663 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T18:13:39,663 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:13:39,663 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T18:13:39,663 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T18:13:39,666 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-07T18:13:39,668 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T18:13:39,668 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:39,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:13:39,670 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-07T18:13:39,671 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T18:13:39,671 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:39,672 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:13:39,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-07T18:13:39,674 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T18:13:39,674 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:39,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:13:39,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-07T18:13:39,677 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T18:13:39,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:39,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T18:13:39,678 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T18:13:39,679 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740 2024-12-07T18:13:39,682 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740 2024-12-07T18:13:39,684 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T18:13:39,684 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T18:13:39,685 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:13:39,687 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T18:13:39,689 INFO [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69564552, jitterRate=0.03659260272979736}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:13:39,689 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T18:13:39,691 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733595219664Writing region info on filesystem at 1733595219664Initializing all the Stores at 1733595219666 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733595219666Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733595219666Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595219666Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733595219666Cleaning up temporary data from old regions at 1733595219684 (+18 ms)Running coprocessor post-open hooks at 1733595219689 (+5 ms)Region opened successfully at 1733595219691 (+2 ms) 2024-12-07T18:13:39,698 INFO [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733595219599 2024-12-07T18:13:39,710 DEBUG [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T18:13:39,710 INFO [RS_OPEN_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T18:13:39,712 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:39,714 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 92acd7e2389b,33947,1733595217075, state=OPEN 2024-12-07T18:13:39,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T18:13:39,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T18:13:39,723 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:13:39,723 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T18:13:39,723 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:39,728 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T18:13:39,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=92acd7e2389b,33947,1733595217075 in 294 msec 2024-12-07T18:13:39,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T18:13:39,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 499 msec 2024-12-07T18:13:39,738 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T18:13:39,738 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T18:13:39,760 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:39,762 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:39,785 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:39,788 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36693, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:39,810 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1340 sec 2024-12-07T18:13:39,810 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733595219810, completionTime=-1 2024-12-07T18:13:39,813 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-07T18:13:39,813 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T18:13:39,870 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-07T18:13:39,870 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733595279870 2024-12-07T18:13:39,870 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733595339870 2024-12-07T18:13:39,870 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 57 msec 2024-12-07T18:13:39,872 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=92acd7e2389b,38531,1733595216342-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:39,873 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=92acd7e2389b,38531,1733595216342-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:39,873 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=92acd7e2389b,38531,1733595216342-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:39,874 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-92acd7e2389b:38531, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:39,875 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:39,875 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:39,881 DEBUG [master/92acd7e2389b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T18:13:39,901 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.573sec 2024-12-07T18:13:39,902 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T18:13:39,903 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T18:13:39,904 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T18:13:39,905 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T18:13:39,905 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T18:13:39,905 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=92acd7e2389b,38531,1733595216342-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T18:13:39,906 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=92acd7e2389b,38531,1733595216342-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T18:13:39,914 DEBUG [master/92acd7e2389b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T18:13:39,914 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T18:13:39,915 INFO [master/92acd7e2389b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=92acd7e2389b,38531,1733595216342-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T18:13:39,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:39,959 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-07T18:13:39,959 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-07T18:13:39,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:39,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:39,978 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45425200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:39,991 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:39,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:39,995 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:40,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:40,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=92acd7e2389b,38531,1733595216342 2024-12-07T18:13:40,042 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=6979 2024-12-07T18:13:40,053 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T18:13:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 92acd7e2389b,38531,1733595216342 2024-12-07T18:13:40,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28aa44c3 2024-12-07T18:13:40,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T18:13:40,066 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T18:13:40,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T18:13:40,073 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:13:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T18:13:40,085 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T18:13:40,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-07T18:13:40,088 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:40,091 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T18:13:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:13:40,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741835_1011 (size=1061) 2024-12-07T18:13:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:13:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:13:40,508 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68 2024-12-07T18:13:40,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741836_1012 (size=53) 2024-12-07T18:13:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:13:40,919 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:13:40,920 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 66753c009d64518d3f4dfefd580afd72, disabling compactions & flushes 2024-12-07T18:13:40,920 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:40,920 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:40,920 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. after waiting 0 ms 2024-12-07T18:13:40,920 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:40,920 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:40,920 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 66753c009d64518d3f4dfefd580afd72: Waiting for close lock at 1733595220920Disabling compacts and flushes for region at 1733595220920Disabling writes for close at 1733595220920Writing region close event to WAL at 1733595220920Closed at 1733595220920 2024-12-07T18:13:40,923 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T18:13:40,929 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733595220923"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733595220923"}]},"ts":"1733595220923"} 2024-12-07T18:13:40,935 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T18:13:40,938 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T18:13:40,941 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595220938"}]},"ts":"1733595220938"} 2024-12-07T18:13:40,947 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T18:13:40,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66753c009d64518d3f4dfefd580afd72, ASSIGN}] 2024-12-07T18:13:40,952 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66753c009d64518d3f4dfefd580afd72, ASSIGN 2024-12-07T18:13:40,954 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66753c009d64518d3f4dfefd580afd72, ASSIGN; state=OFFLINE, location=92acd7e2389b,33947,1733595217075; forceNewPlan=false, retain=false 2024-12-07T18:13:41,106 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=66753c009d64518d3f4dfefd580afd72, regionState=OPENING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:41,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66753c009d64518d3f4dfefd580afd72, ASSIGN because future has completed 2024-12-07T18:13:41,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:13:41,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:13:41,275 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:41,275 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:13:41,276 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,276 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:13:41,277 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,277 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,279 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,283 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:13:41,284 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66753c009d64518d3f4dfefd580afd72 columnFamilyName A 2024-12-07T18:13:41,284 DEBUG [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:41,285 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] regionserver.HStore(327): Store=66753c009d64518d3f4dfefd580afd72/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:13:41,285 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,287 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:13:41,288 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66753c009d64518d3f4dfefd580afd72 columnFamilyName B 2024-12-07T18:13:41,288 DEBUG [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:41,289 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] regionserver.HStore(327): Store=66753c009d64518d3f4dfefd580afd72/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:13:41,289 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,292 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:13:41,292 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66753c009d64518d3f4dfefd580afd72 columnFamilyName C 2024-12-07T18:13:41,293 DEBUG [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:13:41,293 INFO [StoreOpener-66753c009d64518d3f4dfefd580afd72-1 {}] regionserver.HStore(327): Store=66753c009d64518d3f4dfefd580afd72/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:13:41,294 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:41,294 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,296 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,297 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,299 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,299 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,300 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:13:41,303 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,308 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:13:41,309 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 66753c009d64518d3f4dfefd580afd72; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72355382, jitterRate=0.07817921042442322}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:13:41,309 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:41,310 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 66753c009d64518d3f4dfefd580afd72: Running coprocessor pre-open hook at 1733595221277Writing region info on filesystem at 1733595221277Initializing all the Stores at 1733595221279 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595221279Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595221279Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595221279Cleaning up temporary data from old regions at 1733595221299 (+20 ms)Running coprocessor post-open hooks at 1733595221309 (+10 ms)Region opened successfully at 1733595221310 (+1 ms) 2024-12-07T18:13:41,312 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., pid=6, masterSystemTime=1733595221267 2024-12-07T18:13:41,316 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:41,317 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:41,318 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=66753c009d64518d3f4dfefd580afd72, regionState=OPEN, openSeqNum=2, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:41,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:13:41,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T18:13:41,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 in 213 msec 2024-12-07T18:13:41,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T18:13:41,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66753c009d64518d3f4dfefd580afd72, ASSIGN in 381 msec 2024-12-07T18:13:41,336 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T18:13:41,336 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595221336"}]},"ts":"1733595221336"} 2024-12-07T18:13:41,340 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T18:13:41,342 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T18:13:41,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2670 sec 2024-12-07T18:13:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T18:13:42,250 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:13:42,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:42,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:42,267 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253618fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:42,269 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:42,271 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:42,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67dec8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,274 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:42,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:42,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:42,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:42,297 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T18:13:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 92acd7e2389b,38531,1733595216342 2024-12-07T18:13:42,299 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1df1000 2024-12-07T18:13:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T18:13:42,306 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T18:13:42,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:42,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:42,313 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21add1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:42,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d4406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:42,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:42,319 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@450047aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:42,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:42,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:42,326 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:42,330 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:42,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:42,333 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:42,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:42,336 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e0b13b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:42,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:42,339 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f95c81e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:42,342 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f9272c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:42,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:42,345 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:42,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d97ae23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:42,348 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71e49db8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:42,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:42,351 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74deee7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:42,353 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:42,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e021fe2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,364 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,364 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:42,364 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:42,366 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@905b4d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:42,368 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:42,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526369d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,370 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:13:42,370 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:13:42,371 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:13:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:13:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:13:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ce558b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:13:42,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:13:42,373 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:13:42,374 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:13:42,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280a2bc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:13:42,381 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:42,381 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:42,381 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:42,381 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:42,381 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:42,382 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:42,383 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:42,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:42,384 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:42,384 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:42,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:42,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:42,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33392, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:42,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:42,384 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:13:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:42,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:42,386 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:42,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:42,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:42,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:42,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:13:42,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:42,390 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:42,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:13:42,391 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:13:42,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:13:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:13:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:13:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:13:42,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:13:42,400 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:13:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:13:42,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:13:42,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:13:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:13:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-07T18:13:42,410 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:13:42,412 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:13:42,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T18:13:42,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:13:42,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:42,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:13:42,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:42,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:42,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:42,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:42,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:42,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:42,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T18:13:42,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/cfa12dec4f30457a920c7b1e6a53b18e is 50, key is test_row_0/A:col10/1733595222483/Put/seqid=0 2024-12-07T18:13:42,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:42,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:42,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:42,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:42,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:42,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:42,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741837_1013 (size=12001) 2024-12-07T18:13:42,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/cfa12dec4f30457a920c7b1e6a53b18e 2024-12-07T18:13:42,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595282637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:42,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595282647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:42,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595282655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:42,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595282669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:42,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595222616, max timestamp=1733595222661Num uniques 859; 2024-12-07T18:13:42,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595222616, max timestamp=1733595222661Num uniques 859; 2024-12-07T18:13:42,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:13:42,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:13:42,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595222616, max timestamp=1733595222661Num uniques 859; 2024-12-07T18:13:42,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:13:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T18:13:42,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:42,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:42,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:42,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:42,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:42,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:42,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/7df2c2abb5654b97bbc6540b2e1db4a6 is 50, key is test_row_0/B:col10/1733595222483/Put/seqid=0 2024-12-07T18:13:42,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741838_1014 (size=12001) 2024-12-07T18:13:42,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595282894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:42,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595282899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:42,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595282917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:42,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595282917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:42,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,933 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:42,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595282924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:42,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:42,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:42,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:42,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:42,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:42,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:42,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595283030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595283030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595283030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T18:13:43,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595283039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595283040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:43,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/7df2c2abb5654b97bbc6540b2e1db4a6 2024-12-07T18:13:43,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595283240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595283240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595283241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595283250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595283269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,281 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/2110d0c13b8e47c8a4600b1c7d6cebba is 50, key is test_row_0/C:col10/1733595222483/Put/seqid=0 2024-12-07T18:13:43,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741839_1015 (size=12001) 2024-12-07T18:13:43,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/2110d0c13b8e47c8a4600b1c7d6cebba 2024-12-07T18:13:43,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/cfa12dec4f30457a920c7b1e6a53b18e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cfa12dec4f30457a920c7b1e6a53b18e 2024-12-07T18:13:43,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cfa12dec4f30457a920c7b1e6a53b18e, entries=150, sequenceid=13, filesize=11.7 K 2024-12-07T18:13:43,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/7df2c2abb5654b97bbc6540b2e1db4a6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/7df2c2abb5654b97bbc6540b2e1db4a6 2024-12-07T18:13:43,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/7df2c2abb5654b97bbc6540b2e1db4a6, entries=150, sequenceid=13, filesize=11.7 K 2024-12-07T18:13:43,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/2110d0c13b8e47c8a4600b1c7d6cebba as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2110d0c13b8e47c8a4600b1c7d6cebba 2024-12-07T18:13:43,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2110d0c13b8e47c8a4600b1c7d6cebba, entries=150, sequenceid=13, filesize=11.7 K 2024-12-07T18:13:43,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:43,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=134.18 KB/137400 for 66753c009d64518d3f4dfefd580afd72 in 961ms, sequenceid=13, compaction requested=false 2024-12-07T18:13:43,447 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-07T18:13:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:43,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T18:13:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:43,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-07T18:13:43,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:43,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733595222886, max timestamp=1733595223562Num uniques 1018; 2024-12-07T18:13:43,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-07T18:13:43,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T18:13:43,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:43,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:43,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:43,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T18:13:43,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:43,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:43,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/728c0ff86fc643b9992e3edf9c1666e8 is 50, key is test_row_0/A:col10/1733595223562/Put/seqid=0 2024-12-07T18:13:43,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741840_1016 (size=14341) 2024-12-07T18:13:43,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/728c0ff86fc643b9992e3edf9c1666e8 2024-12-07T18:13:43,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/b4c6a44c3f044d55b1203629765f3781 is 50, key is test_row_0/B:col10/1733595223562/Put/seqid=0 2024-12-07T18:13:43,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595283698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595283703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595283708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741841_1017 (size=12001) 2024-12-07T18:13:43,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/b4c6a44c3f044d55b1203629765f3781 2024-12-07T18:13:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595283723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595283723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:43,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:43,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/c841f041d6314887a8246ea8b33dbeb6 is 50, key is test_row_0/C:col10/1733595223562/Put/seqid=0 2024-12-07T18:13:43,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741842_1018 (size=12001) 2024-12-07T18:13:43,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595283829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595283830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595283839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595283869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:43,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595283869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:43,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:43,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:43,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:43,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:43,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595284048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595284059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595284069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:44,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595284090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595284090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/c841f041d6314887a8246ea8b33dbeb6 2024-12-07T18:13:44,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/728c0ff86fc643b9992e3edf9c1666e8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/728c0ff86fc643b9992e3edf9c1666e8 2024-12-07T18:13:44,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/728c0ff86fc643b9992e3edf9c1666e8, entries=200, sequenceid=52, filesize=14.0 K 2024-12-07T18:13:44,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/b4c6a44c3f044d55b1203629765f3781 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b4c6a44c3f044d55b1203629765f3781 2024-12-07T18:13:44,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:44,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:44,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b4c6a44c3f044d55b1203629765f3781, entries=150, sequenceid=52, filesize=11.7 K 2024-12-07T18:13:44,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/c841f041d6314887a8246ea8b33dbeb6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/c841f041d6314887a8246ea8b33dbeb6 2024-12-07T18:13:44,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/c841f041d6314887a8246ea8b33dbeb6, entries=150, sequenceid=52, filesize=11.7 K 2024-12-07T18:13:44,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~109.58 KB/112210, heapSize ~312.08 KB/319568, currentSize=67.09 KB/68700 for 66753c009d64518d3f4dfefd580afd72 in 692ms, sequenceid=52, compaction requested=false 2024-12-07T18:13:44,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:44,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T18:13:44,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:44,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:44,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:44,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:44,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:44,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:44,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/752ed38067884ab788e1a1633d6b1ae8 is 50, key is test_row_0/A:col10/1733595223701/Put/seqid=0 2024-12-07T18:13:44,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:44,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:44,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741843_1019 (size=12001) 2024-12-07T18:13:44,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595284422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595284424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595284425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595284427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595284430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595284539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595284539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595284539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:44,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:44,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595284549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595284550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T18:13:44,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595284749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595284749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595284750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595284769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595284770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:44,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/752ed38067884ab788e1a1633d6b1ae8 2024-12-07T18:13:44,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/6dda319301654397a38ee5f837af986a is 50, key is test_row_0/B:col10/1733595223701/Put/seqid=0 2024-12-07T18:13:44,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741844_1020 (size=12001) 2024-12-07T18:13:44,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:44,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:44,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:45,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:45,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:45,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:45,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595285059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595285059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595285069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595285078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595285079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,173 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T18:13:45,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:45,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:45,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:45,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:45,234 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-07T18:13:45,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/6dda319301654397a38ee5f837af986a 2024-12-07T18:13:45,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/a569cedd82c747d88c4c88af141afd1a is 50, key is test_row_0/C:col10/1733595223701/Put/seqid=0 2024-12-07T18:13:45,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741845_1021 (size=12001) 2024-12-07T18:13:45,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=67 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/a569cedd82c747d88c4c88af141afd1a 2024-12-07T18:13:45,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/752ed38067884ab788e1a1633d6b1ae8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/752ed38067884ab788e1a1633d6b1ae8 2024-12-07T18:13:45,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/752ed38067884ab788e1a1633d6b1ae8, entries=150, sequenceid=67, filesize=11.7 K 2024-12-07T18:13:45,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/6dda319301654397a38ee5f837af986a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6dda319301654397a38ee5f837af986a 2024-12-07T18:13:45,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:45,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:45,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:45,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:45,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:45,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6dda319301654397a38ee5f837af986a, entries=150, sequenceid=67, filesize=11.7 K 2024-12-07T18:13:45,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/a569cedd82c747d88c4c88af141afd1a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a569cedd82c747d88c4c88af141afd1a 2024-12-07T18:13:45,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a569cedd82c747d88c4c88af141afd1a, entries=150, sequenceid=67, filesize=11.7 K 2024-12-07T18:13:45,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 66753c009d64518d3f4dfefd580afd72 in 995ms, sequenceid=67, compaction requested=true 2024-12-07T18:13:45,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:45,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:13:45,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:45,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:13:45,381 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:45,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:45,381 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:45,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:13:45,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:45,388 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:45,390 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:13:45,390 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,391 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/7df2c2abb5654b97bbc6540b2e1db4a6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b4c6a44c3f044d55b1203629765f3781, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6dda319301654397a38ee5f837af986a] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=35.2 K 2024-12-07T18:13:45,392 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:45,392 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df2c2abb5654b97bbc6540b2e1db4a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733595222472 2024-12-07T18:13:45,393 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:13:45,393 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,393 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting b4c6a44c3f044d55b1203629765f3781, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733595222886 2024-12-07T18:13:45,394 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cfa12dec4f30457a920c7b1e6a53b18e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/728c0ff86fc643b9992e3edf9c1666e8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/752ed38067884ab788e1a1633d6b1ae8] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=37.4 K 2024-12-07T18:13:45,394 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dda319301654397a38ee5f837af986a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733595223701 2024-12-07T18:13:45,401 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfa12dec4f30457a920c7b1e6a53b18e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733595222472 2024-12-07T18:13:45,403 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 728c0ff86fc643b9992e3edf9c1666e8, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733595222886 2024-12-07T18:13:45,404 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 752ed38067884ab788e1a1633d6b1ae8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733595223701 2024-12-07T18:13:45,455 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:45,456 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#9 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:45,457 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/cb1b3cbef1884ff49abd74ea62371386 is 50, key is test_row_0/A:col10/1733595223701/Put/seqid=0 2024-12-07T18:13:45,457 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/97f3fdc69de74e05b28fb99a62fcb98b is 50, key is test_row_0/B:col10/1733595223701/Put/seqid=0 2024-12-07T18:13:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741847_1023 (size=12104) 2024-12-07T18:13:45,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741846_1022 (size=12104) 2024-12-07T18:13:45,487 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/cb1b3cbef1884ff49abd74ea62371386 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cb1b3cbef1884ff49abd74ea62371386 2024-12-07T18:13:45,491 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/97f3fdc69de74e05b28fb99a62fcb98b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/97f3fdc69de74e05b28fb99a62fcb98b 2024-12-07T18:13:45,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-07T18:13:45,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,498 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-07T18:13:45,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:45,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:45,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:45,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:45,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:45,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:45,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/353bc3fbc8a7493baca021ef0e4fb86e is 50, key is test_row_0/A:col10/1733595224402/Put/seqid=0 2024-12-07T18:13:45,527 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into cb1b3cbef1884ff49abd74ea62371386(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:45,527 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into 97f3fdc69de74e05b28fb99a62fcb98b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:45,527 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:45,527 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:45,531 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595225381; duration=0sec 2024-12-07T18:13:45,531 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595225373; duration=0sec 2024-12-07T18:13:45,531 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:45,531 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:13:45,531 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:45,534 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:45,534 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:13:45,534 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,534 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2110d0c13b8e47c8a4600b1c7d6cebba, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/c841f041d6314887a8246ea8b33dbeb6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a569cedd82c747d88c4c88af141afd1a] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=35.2 K 2024-12-07T18:13:45,535 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 2110d0c13b8e47c8a4600b1c7d6cebba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1733595222472 2024-12-07T18:13:45,536 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting c841f041d6314887a8246ea8b33dbeb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733595222886 2024-12-07T18:13:45,537 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting a569cedd82c747d88c4c88af141afd1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733595223701 2024-12-07T18:13:45,539 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:45,539 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:13:45,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741848_1024 (size=12001) 2024-12-07T18:13:45,557 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/353bc3fbc8a7493baca021ef0e4fb86e 2024-12-07T18:13:45,565 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:45,566 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/322d0ba18d4a4aa1bf3049d0d34b77ab is 50, key is test_row_0/C:col10/1733595223701/Put/seqid=0 2024-12-07T18:13:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:45,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:45,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/c42e65275d134983b72f6ac1fc13265f is 50, key is test_row_0/B:col10/1733595224402/Put/seqid=0 2024-12-07T18:13:45,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741849_1025 (size=12104) 2024-12-07T18:13:45,619 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/322d0ba18d4a4aa1bf3049d0d34b77ab as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/322d0ba18d4a4aa1bf3049d0d34b77ab 2024-12-07T18:13:45,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595285609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595285611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595285614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741850_1026 (size=12001) 2024-12-07T18:13:45,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595285620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,633 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/c42e65275d134983b72f6ac1fc13265f 2024-12-07T18:13:45,638 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into 322d0ba18d4a4aa1bf3049d0d34b77ab(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:45,638 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:45,638 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595225381; duration=0sec 2024-12-07T18:13:45,638 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:45,638 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:13:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595285618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ae0ac7964c654fc29cf30f1e1c441de8 is 50, key is test_row_0/C:col10/1733595224402/Put/seqid=0 2024-12-07T18:13:45,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741851_1027 (size=12001) 2024-12-07T18:13:45,687 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ae0ac7964c654fc29cf30f1e1c441de8 2024-12-07T18:13:45,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/353bc3fbc8a7493baca021ef0e4fb86e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/353bc3fbc8a7493baca021ef0e4fb86e 2024-12-07T18:13:45,716 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/353bc3fbc8a7493baca021ef0e4fb86e, entries=150, sequenceid=84, filesize=11.7 K 2024-12-07T18:13:45,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/c42e65275d134983b72f6ac1fc13265f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/c42e65275d134983b72f6ac1fc13265f 2024-12-07T18:13:45,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595285729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595285729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,736 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/c42e65275d134983b72f6ac1fc13265f, entries=150, sequenceid=84, filesize=11.7 K 2024-12-07T18:13:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595285729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ae0ac7964c654fc29cf30f1e1c441de8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ae0ac7964c654fc29cf30f1e1c441de8 2024-12-07T18:13:45,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595285738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,751 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ae0ac7964c654fc29cf30f1e1c441de8, entries=150, sequenceid=84, filesize=11.7 K 2024-12-07T18:13:45,753 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 66753c009d64518d3f4dfefd580afd72 in 256ms, sequenceid=84, compaction requested=false 2024-12-07T18:13:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:45,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-07T18:13:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-07T18:13:45,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T18:13:45,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:45,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:45,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:45,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:45,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:45,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:45,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T18:13:45,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3420 sec 2024-12-07T18:13:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:45,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.3610 sec 2024-12-07T18:13:45,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/35546658084743c1bc99600714b481e9 is 50, key is test_row_0/A:col10/1733595225613/Put/seqid=0 2024-12-07T18:13:45,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741852_1028 (size=14341) 2024-12-07T18:13:45,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/35546658084743c1bc99600714b481e9 2024-12-07T18:13:45,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/590245b75cfd4185b2a061a27d825f82 is 50, key is test_row_0/B:col10/1733595225613/Put/seqid=0 2024-12-07T18:13:45,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595285887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741853_1029 (size=12001) 2024-12-07T18:13:45,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/590245b75cfd4185b2a061a27d825f82 2024-12-07T18:13:45,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/5ab7901088ca439ebfdc47a03aabafe6 is 50, key is test_row_0/C:col10/1733595225613/Put/seqid=0 2024-12-07T18:13:45,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741854_1030 (size=12001) 2024-12-07T18:13:45,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/5ab7901088ca439ebfdc47a03aabafe6 2024-12-07T18:13:45,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595285949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595285950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595285950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595285950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:45,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:45,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/35546658084743c1bc99600714b481e9 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/35546658084743c1bc99600714b481e9 2024-12-07T18:13:45,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/35546658084743c1bc99600714b481e9, entries=200, sequenceid=103, filesize=14.0 K 2024-12-07T18:13:45,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/590245b75cfd4185b2a061a27d825f82 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/590245b75cfd4185b2a061a27d825f82 2024-12-07T18:13:45,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/590245b75cfd4185b2a061a27d825f82, entries=150, sequenceid=103, filesize=11.7 K 2024-12-07T18:13:46,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/5ab7901088ca439ebfdc47a03aabafe6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/5ab7901088ca439ebfdc47a03aabafe6 2024-12-07T18:13:46,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595285999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/5ab7901088ca439ebfdc47a03aabafe6, entries=150, sequenceid=103, filesize=11.7 K 2024-12-07T18:13:46,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 66753c009d64518d3f4dfefd580afd72 in 268ms, sequenceid=103, compaction requested=true 2024-12-07T18:13:46,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:46,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:13:46,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:46,028 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:46,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:13:46,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:46,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:13:46,028 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:46,028 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:46,032 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:46,032 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:46,032 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:13:46,032 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:13:46,032 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:46,032 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:46,033 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/97f3fdc69de74e05b28fb99a62fcb98b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/c42e65275d134983b72f6ac1fc13265f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/590245b75cfd4185b2a061a27d825f82] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=35.3 K 2024-12-07T18:13:46,033 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cb1b3cbef1884ff49abd74ea62371386, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/353bc3fbc8a7493baca021ef0e4fb86e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/35546658084743c1bc99600714b481e9] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=37.5 K 2024-12-07T18:13:46,034 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f3fdc69de74e05b28fb99a62fcb98b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733595223701 2024-12-07T18:13:46,034 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1b3cbef1884ff49abd74ea62371386, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733595223701 2024-12-07T18:13:46,035 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting c42e65275d134983b72f6ac1fc13265f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733595224402 2024-12-07T18:13:46,035 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 353bc3fbc8a7493baca021ef0e4fb86e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733595224402 2024-12-07T18:13:46,036 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 590245b75cfd4185b2a061a27d825f82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733595225613 2024-12-07T18:13:46,036 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35546658084743c1bc99600714b481e9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733595225610 2024-12-07T18:13:46,061 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#18 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:46,062 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/06a43bf855994b3eacd151248c99fa54 is 50, key is test_row_0/B:col10/1733595225613/Put/seqid=0 2024-12-07T18:13:46,069 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:46,070 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/28a2e705c281427fa02b4ae141bc4625 is 50, key is test_row_0/A:col10/1733595225613/Put/seqid=0 2024-12-07T18:13:46,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741855_1031 (size=12207) 2024-12-07T18:13:46,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741856_1032 (size=12207) 2024-12-07T18:13:46,104 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/06a43bf855994b3eacd151248c99fa54 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/06a43bf855994b3eacd151248c99fa54 2024-12-07T18:13:46,105 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/28a2e705c281427fa02b4ae141bc4625 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28a2e705c281427fa02b4ae141bc4625 2024-12-07T18:13:46,120 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into 06a43bf855994b3eacd151248c99fa54(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:46,120 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:46,120 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595226028; duration=0sec 2024-12-07T18:13:46,121 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:46,121 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:13:46,121 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:46,126 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:46,126 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:13:46,126 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:46,126 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into 28a2e705c281427fa02b4ae141bc4625(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:46,126 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:46,126 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595226028; duration=0sec 2024-12-07T18:13:46,127 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:46,127 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:13:46,127 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/322d0ba18d4a4aa1bf3049d0d34b77ab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ae0ac7964c654fc29cf30f1e1c441de8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/5ab7901088ca439ebfdc47a03aabafe6] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=35.3 K 2024-12-07T18:13:46,128 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 322d0ba18d4a4aa1bf3049d0d34b77ab, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=67, earliestPutTs=1733595223701 2024-12-07T18:13:46,130 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0ac7964c654fc29cf30f1e1c441de8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1733595224402 2024-12-07T18:13:46,131 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ab7901088ca439ebfdc47a03aabafe6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733595225613 2024-12-07T18:13:46,174 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#20 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:46,175 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/e41922ab38a740ba908ace4dae771150 is 50, key is test_row_0/C:col10/1733595225613/Put/seqid=0 2024-12-07T18:13:46,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:46,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-07T18:13:46,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:46,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:46,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741857_1033 (size=12207) 2024-12-07T18:13:46,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/d6dee20ff4c64c2896695fb8e90485e0 is 50, key is test_row_0/A:col10/1733595225863/Put/seqid=0 2024-12-07T18:13:46,246 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/e41922ab38a740ba908ace4dae771150 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/e41922ab38a740ba908ace4dae771150 2024-12-07T18:13:46,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741858_1034 (size=14341) 2024-12-07T18:13:46,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/d6dee20ff4c64c2896695fb8e90485e0 2024-12-07T18:13:46,278 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into e41922ab38a740ba908ace4dae771150(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:46,279 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:46,279 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595226028; duration=0sec 2024-12-07T18:13:46,279 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:46,279 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:13:46,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/e047e9b0853e42be90393854bd7bf3ea is 50, key is test_row_0/B:col10/1733595225863/Put/seqid=0 2024-12-07T18:13:46,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595286285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595286290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595286297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595286301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595286299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741859_1035 (size=12001) 2024-12-07T18:13:46,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/e047e9b0853e42be90393854bd7bf3ea 2024-12-07T18:13:46,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/dbcf4e86b0064b668841c8a76a17e4de is 50, key is test_row_0/C:col10/1733595225863/Put/seqid=0 2024-12-07T18:13:46,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741860_1036 (size=12001) 2024-12-07T18:13:46,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595286409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595286429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595286429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595286429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595286430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-07T18:13:46,569 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:13:46,570 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:13:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-07T18:13:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-07T18:13:46,576 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:13:46,579 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:13:46,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:13:46,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595286618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595286639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595286639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595286640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595286640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-07T18:13:46,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-07T18:13:46,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:46,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:46,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:46,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:46,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:46,751 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-07T18:13:46,752 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-07T18:13:46,754 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T18:13:46,754 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-07T18:13:46,755 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-07T18:13:46,755 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-07T18:13:46,756 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-07T18:13:46,756 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-07T18:13:46,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/dbcf4e86b0064b668841c8a76a17e4de 2024-12-07T18:13:46,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/d6dee20ff4c64c2896695fb8e90485e0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d6dee20ff4c64c2896695fb8e90485e0 2024-12-07T18:13:46,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d6dee20ff4c64c2896695fb8e90485e0, entries=200, sequenceid=122, filesize=14.0 K 2024-12-07T18:13:46,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/e047e9b0853e42be90393854bd7bf3ea as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e047e9b0853e42be90393854bd7bf3ea 2024-12-07T18:13:46,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e047e9b0853e42be90393854bd7bf3ea, entries=150, sequenceid=122, filesize=11.7 K 2024-12-07T18:13:46,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/dbcf4e86b0064b668841c8a76a17e4de as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/dbcf4e86b0064b668841c8a76a17e4de 2024-12-07T18:13:46,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/dbcf4e86b0064b668841c8a76a17e4de, entries=150, sequenceid=122, filesize=11.7 K 2024-12-07T18:13:46,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 66753c009d64518d3f4dfefd580afd72 in 651ms, sequenceid=122, compaction requested=false 2024-12-07T18:13:46,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-07T18:13:46,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-07T18:13:46,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:46,889 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T18:13:46,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:46,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:46,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:46,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:46,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:46,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/f1dfad50f4b04a29b13aa5c916ba5300 is 50, key is test_row_0/A:col10/1733595226289/Put/seqid=0 2024-12-07T18:13:46,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:46,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:46,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741861_1037 (size=12151) 2024-12-07T18:13:46,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595286982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595286985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595286988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:46,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:46,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595286990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:46,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:46,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595286992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595287099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595287099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595287108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595287109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595287111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-07T18:13:47,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595287309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595287318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595287319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595287320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595287328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,359 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/f1dfad50f4b04a29b13aa5c916ba5300 2024-12-07T18:13:47,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/400ae14df2a641ddb3b97887e6524bd2 is 50, key is test_row_0/B:col10/1733595226289/Put/seqid=0 2024-12-07T18:13:47,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741862_1038 (size=12151) 2024-12-07T18:13:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595287619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595287628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595287628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595287639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595287639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:47,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-07T18:13:47,787 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/400ae14df2a641ddb3b97887e6524bd2 2024-12-07T18:13:47,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/563da3eaa9d549388a6b4b9f02690af0 is 50, key is test_row_0/C:col10/1733595226289/Put/seqid=0 2024-12-07T18:13:47,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741863_1039 (size=12151) 2024-12-07T18:13:47,816 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/563da3eaa9d549388a6b4b9f02690af0 2024-12-07T18:13:47,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/f1dfad50f4b04a29b13aa5c916ba5300 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/f1dfad50f4b04a29b13aa5c916ba5300 2024-12-07T18:13:47,836 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/f1dfad50f4b04a29b13aa5c916ba5300, entries=150, sequenceid=138, filesize=11.9 K 2024-12-07T18:13:47,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/400ae14df2a641ddb3b97887e6524bd2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/400ae14df2a641ddb3b97887e6524bd2 2024-12-07T18:13:47,847 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/400ae14df2a641ddb3b97887e6524bd2, entries=150, sequenceid=138, filesize=11.9 K 2024-12-07T18:13:47,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/563da3eaa9d549388a6b4b9f02690af0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/563da3eaa9d549388a6b4b9f02690af0 2024-12-07T18:13:47,863 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/563da3eaa9d549388a6b4b9f02690af0, entries=150, sequenceid=138, filesize=11.9 K 2024-12-07T18:13:47,868 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 66753c009d64518d3f4dfefd580afd72 in 979ms, sequenceid=138, compaction requested=true 2024-12-07T18:13:47,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:47,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:47,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-07T18:13:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-07T18:13:47,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-07T18:13:47,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2920 sec 2024-12-07T18:13:47,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3100 sec 2024-12-07T18:13:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:48,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T18:13:48,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:48,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:48,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:48,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:48,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:48,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:48,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/b7a2c42d304e4cbb966ff43f124e951a is 50, key is test_row_0/A:col10/1733595228132/Put/seqid=0 2024-12-07T18:13:48,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595288155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595288158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595288158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595288159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595288161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741864_1040 (size=14541) 2024-12-07T18:13:48,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595288269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595288269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595288269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595288269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595288479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595288479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595288480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595288480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/b7a2c42d304e4cbb966ff43f124e951a 2024-12-07T18:13:48,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/3786526879354832b133d4242e18771e is 50, key is test_row_0/B:col10/1733595228132/Put/seqid=0 2024-12-07T18:13:48,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741865_1041 (size=12151) 2024-12-07T18:13:48,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-07T18:13:48,720 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:13:48,720 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:13:48,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-07T18:13:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-07T18:13:48,727 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:13:48,729 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:13:48,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:13:48,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595288789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595288790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595288798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:48,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595288798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:48,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-07T18:13:48,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-07T18:13:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:48,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:48,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:48,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:49,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/3786526879354832b133d4242e18771e 2024-12-07T18:13:49,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/cb1dbe2ca41744d0b528c1e941face92 is 50, key is test_row_0/C:col10/1733595228132/Put/seqid=0 2024-12-07T18:13:49,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-07T18:13:49,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-07T18:13:49,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:49,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:49,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:49,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:49,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:49,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741866_1042 (size=12151) 2024-12-07T18:13:49,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595289179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:49,184 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 66753c009d64518d3f4dfefd580afd72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:13:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:49,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-07T18:13:49,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:49,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:49,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:49,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:49,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:49,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595289308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:49,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595289308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:49,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595289308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:49,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:49,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:49,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595289319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:49,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:49,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:49,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:49,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-07T18:13:49,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-07T18:13:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:49,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:49,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:49,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:49,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/cb1dbe2ca41744d0b528c1e941face92 2024-12-07T18:13:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/b7a2c42d304e4cbb966ff43f124e951a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b7a2c42d304e4cbb966ff43f124e951a 2024-12-07T18:13:49,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b7a2c42d304e4cbb966ff43f124e951a, entries=200, sequenceid=157, filesize=14.2 K 2024-12-07T18:13:49,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/3786526879354832b133d4242e18771e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/3786526879354832b133d4242e18771e 2024-12-07T18:13:49,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/3786526879354832b133d4242e18771e, entries=150, sequenceid=157, filesize=11.9 K 2024-12-07T18:13:49,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/cb1dbe2ca41744d0b528c1e941face92 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/cb1dbe2ca41744d0b528c1e941face92 2024-12-07T18:13:49,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/cb1dbe2ca41744d0b528c1e941face92, entries=150, sequenceid=157, filesize=11.9 K 2024-12-07T18:13:49,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 66753c009d64518d3f4dfefd580afd72 in 1355ms, sequenceid=157, compaction requested=true 2024-12-07T18:13:49,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:13:49,489 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:49,489 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:13:49,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:49,491 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53240 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:13:49,491 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:13:49,491 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:13:49,491 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:13:49,491 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:49,491 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:49,491 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28a2e705c281427fa02b4ae141bc4625, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d6dee20ff4c64c2896695fb8e90485e0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/f1dfad50f4b04a29b13aa5c916ba5300, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b7a2c42d304e4cbb966ff43f124e951a] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=52.0 K 2024-12-07T18:13:49,491 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/06a43bf855994b3eacd151248c99fa54, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e047e9b0853e42be90393854bd7bf3ea, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/400ae14df2a641ddb3b97887e6524bd2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/3786526879354832b133d4242e18771e] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=47.4 K 2024-12-07T18:13:49,492 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a2e705c281427fa02b4ae141bc4625, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733595225613 2024-12-07T18:13:49,492 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 06a43bf855994b3eacd151248c99fa54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733595225613 2024-12-07T18:13:49,493 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e047e9b0853e42be90393854bd7bf3ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733595225863 2024-12-07T18:13:49,493 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6dee20ff4c64c2896695fb8e90485e0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733595225863 2024-12-07T18:13:49,494 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 400ae14df2a641ddb3b97887e6524bd2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733595226289 2024-12-07T18:13:49,494 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1dfad50f4b04a29b13aa5c916ba5300, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733595226289 2024-12-07T18:13:49,494 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3786526879354832b133d4242e18771e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733595226985 2024-12-07T18:13:49,495 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7a2c42d304e4cbb966ff43f124e951a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733595226982 2024-12-07T18:13:49,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-07T18:13:49,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:49,507 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T18:13:49,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/9e99d9a979124ecb9854314a5e1ac33d is 50, key is test_row_0/A:col10/1733595228156/Put/seqid=0 2024-12-07T18:13:49,529 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#31 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:49,530 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/57eb56cf845d4aa89b3e26fb390ee7dd is 50, key is test_row_0/B:col10/1733595228132/Put/seqid=0 2024-12-07T18:13:49,534 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:49,535 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/83ba726f19c94a56b50221936ccf3f90 is 50, key is test_row_0/A:col10/1733595228132/Put/seqid=0 2024-12-07T18:13:49,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741868_1044 (size=12493) 2024-12-07T18:13:49,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741867_1043 (size=12151) 2024-12-07T18:13:49,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741869_1045 (size=12493) 2024-12-07T18:13:49,578 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/83ba726f19c94a56b50221936ccf3f90 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/83ba726f19c94a56b50221936ccf3f90 2024-12-07T18:13:49,592 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into 83ba726f19c94a56b50221936ccf3f90(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:49,592 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:49,592 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=12, startTime=1733595229488; duration=0sec 2024-12-07T18:13:49,592 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:49,592 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:13:49,592 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:13:49,594 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:13:49,595 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:13:49,595 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:49,595 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/e41922ab38a740ba908ace4dae771150, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/dbcf4e86b0064b668841c8a76a17e4de, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/563da3eaa9d549388a6b4b9f02690af0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/cb1dbe2ca41744d0b528c1e941face92] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=47.4 K 2024-12-07T18:13:49,596 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting e41922ab38a740ba908ace4dae771150, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1733595225613 2024-12-07T18:13:49,597 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbcf4e86b0064b668841c8a76a17e4de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1733595225863 2024-12-07T18:13:49,599 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 563da3eaa9d549388a6b4b9f02690af0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733595226289 2024-12-07T18:13:49,600 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb1dbe2ca41744d0b528c1e941face92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733595226985 2024-12-07T18:13:49,621 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:49,622 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/a079029c0bbd44fdb6a300ab5b791d62 is 50, key is test_row_0/C:col10/1733595228132/Put/seqid=0 2024-12-07T18:13:49,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741870_1046 (size=12493) 2024-12-07T18:13:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-07T18:13:49,961 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/9e99d9a979124ecb9854314a5e1ac33d 2024-12-07T18:13:49,980 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/57eb56cf845d4aa89b3e26fb390ee7dd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/57eb56cf845d4aa89b3e26fb390ee7dd 2024-12-07T18:13:49,990 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into 57eb56cf845d4aa89b3e26fb390ee7dd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:49,990 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:49,990 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=12, startTime=1733595229489; duration=0sec 2024-12-07T18:13:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/e3406dab37e24e36a72240b052c4570b is 50, key is test_row_0/B:col10/1733595228156/Put/seqid=0 2024-12-07T18:13:49,990 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:49,990 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:13:49,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741871_1047 (size=12151) 2024-12-07T18:13:50,000 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/e3406dab37e24e36a72240b052c4570b 2024-12-07T18:13:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/2fa654a6523c438a866a4a20e0bc6793 is 50, key is test_row_0/C:col10/1733595228156/Put/seqid=0 2024-12-07T18:13:50,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741872_1048 (size=12151) 2024-12-07T18:13:50,057 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/a079029c0bbd44fdb6a300ab5b791d62 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a079029c0bbd44fdb6a300ab5b791d62 2024-12-07T18:13:50,069 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into a079029c0bbd44fdb6a300ab5b791d62(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:50,069 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:50,069 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=12, startTime=1733595229489; duration=0sec 2024-12-07T18:13:50,070 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:50,070 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:13:50,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:50,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:50,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595290353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595290355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595290357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595290357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,436 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/2fa654a6523c438a866a4a20e0bc6793 2024-12-07T18:13:50,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/9e99d9a979124ecb9854314a5e1ac33d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/9e99d9a979124ecb9854314a5e1ac33d 2024-12-07T18:13:50,458 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/9e99d9a979124ecb9854314a5e1ac33d, entries=150, sequenceid=170, filesize=11.9 K 2024-12-07T18:13:50,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/e3406dab37e24e36a72240b052c4570b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e3406dab37e24e36a72240b052c4570b 2024-12-07T18:13:50,470 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e3406dab37e24e36a72240b052c4570b, entries=150, sequenceid=170, filesize=11.9 K 2024-12-07T18:13:50,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/2fa654a6523c438a866a4a20e0bc6793 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2fa654a6523c438a866a4a20e0bc6793 2024-12-07T18:13:50,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595290468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595290469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595290469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595290469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,479 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2fa654a6523c438a866a4a20e0bc6793, entries=150, sequenceid=170, filesize=11.9 K 2024-12-07T18:13:50,481 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 66753c009d64518d3f4dfefd580afd72 in 974ms, sequenceid=170, compaction requested=false 2024-12-07T18:13:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:50,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-07T18:13:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-07T18:13:50,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-07T18:13:50,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7540 sec 2024-12-07T18:13:50,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7680 sec 2024-12-07T18:13:50,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:50,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T18:13:50,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:50,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:50,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:50,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:50,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:50,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:50,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/65b1f66679da43189a444ca854ccdba0 is 50, key is test_row_0/A:col10/1733595230354/Put/seqid=0 2024-12-07T18:13:50,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595290729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595290732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741873_1049 (size=12151) 2024-12-07T18:13:50,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595290735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595290736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595290838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595290838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,842 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595290849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595290849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:50,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-07T18:13:50,869 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:13:50,869 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:13:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-07T18:13:50,872 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:13:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T18:13:50,874 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:13:50,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:13:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T18:13:51,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-07T18:13:51,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:51,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595291048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595291048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595291058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595291058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/65b1f66679da43189a444ca854ccdba0 2024-12-07T18:13:51,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/f46fcff3ddb944f3a731a44782333603 is 50, key is test_row_0/B:col10/1733595230354/Put/seqid=0 2024-12-07T18:13:51,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741874_1050 (size=12151) 2024-12-07T18:13:51,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/f46fcff3ddb944f3a731a44782333603 2024-12-07T18:13:51,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ac2914f7d1b948db8c1a6f32eca099dd is 50, key is test_row_0/C:col10/1733595230354/Put/seqid=0 2024-12-07T18:13:51,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-07T18:13:51,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:51,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T18:13:51,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741875_1051 (size=12151) 2024-12-07T18:13:51,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595291198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,204 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 66753c009d64518d3f4dfefd580afd72 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:13:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-07T18:13:51,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:51,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595291359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595291359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595291368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595291368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,374 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-07T18:13:51,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T18:13:51,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ac2914f7d1b948db8c1a6f32eca099dd 2024-12-07T18:13:51,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/65b1f66679da43189a444ca854ccdba0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/65b1f66679da43189a444ca854ccdba0 2024-12-07T18:13:51,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/65b1f66679da43189a444ca854ccdba0, entries=150, sequenceid=194, filesize=11.9 K 2024-12-07T18:13:51,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/f46fcff3ddb944f3a731a44782333603 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f46fcff3ddb944f3a731a44782333603 2024-12-07T18:13:51,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f46fcff3ddb944f3a731a44782333603, entries=150, sequenceid=194, filesize=11.9 K 2024-12-07T18:13:51,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ac2914f7d1b948db8c1a6f32eca099dd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ac2914f7d1b948db8c1a6f32eca099dd 2024-12-07T18:13:51,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ac2914f7d1b948db8c1a6f32eca099dd, entries=150, sequenceid=194, filesize=11.9 K 2024-12-07T18:13:51,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 66753c009d64518d3f4dfefd580afd72 in 950ms, sequenceid=194, compaction requested=true 2024-12-07T18:13:51,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:51,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:13:51,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:51,633 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:51,633 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:51,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:13:51,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:51,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:13:51,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:51,634 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:51,634 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:51,634 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:13:51,634 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:13:51,635 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,635 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,635 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/57eb56cf845d4aa89b3e26fb390ee7dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e3406dab37e24e36a72240b052c4570b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f46fcff3ddb944f3a731a44782333603] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=35.9 K 2024-12-07T18:13:51,635 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/83ba726f19c94a56b50221936ccf3f90, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/9e99d9a979124ecb9854314a5e1ac33d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/65b1f66679da43189a444ca854ccdba0] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=35.9 K 2024-12-07T18:13:51,636 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 57eb56cf845d4aa89b3e26fb390ee7dd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733595226985 2024-12-07T18:13:51,636 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83ba726f19c94a56b50221936ccf3f90, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733595226985 2024-12-07T18:13:51,636 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e3406dab37e24e36a72240b052c4570b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733595228155 2024-12-07T18:13:51,636 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e99d9a979124ecb9854314a5e1ac33d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733595228155 2024-12-07T18:13:51,637 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting f46fcff3ddb944f3a731a44782333603, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733595230354 2024-12-07T18:13:51,637 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65b1f66679da43189a444ca854ccdba0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733595230354 2024-12-07T18:13:51,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-07T18:13:51,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,647 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T18:13:51,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:51,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:51,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:51,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:51,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:51,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:51,649 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:51,650 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/d4db4f24a6c84866b644bac684709dde is 50, key is test_row_0/B:col10/1733595230354/Put/seqid=0 2024-12-07T18:13:51,650 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:51,651 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/6d2f8ed2a17948e6a0e7ed90d47c334c is 50, key is test_row_0/A:col10/1733595230354/Put/seqid=0 2024-12-07T18:13:51,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/eb6be9b3eb9d40a88f0e339534078613 is 50, key is test_row_0/A:col10/1733595230732/Put/seqid=0 2024-12-07T18:13:51,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741876_1052 (size=12595) 2024-12-07T18:13:51,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741878_1054 (size=12151) 2024-12-07T18:13:51,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741877_1053 (size=12595) 2024-12-07T18:13:51,698 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/6d2f8ed2a17948e6a0e7ed90d47c334c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6d2f8ed2a17948e6a0e7ed90d47c334c 2024-12-07T18:13:51,707 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into 6d2f8ed2a17948e6a0e7ed90d47c334c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:51,707 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:51,707 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595231633; duration=0sec 2024-12-07T18:13:51,707 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:51,707 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:13:51,708 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:51,709 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:51,710 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:13:51,710 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:51,710 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a079029c0bbd44fdb6a300ab5b791d62, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2fa654a6523c438a866a4a20e0bc6793, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ac2914f7d1b948db8c1a6f32eca099dd] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=35.9 K 2024-12-07T18:13:51,710 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting a079029c0bbd44fdb6a300ab5b791d62, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733595226985 2024-12-07T18:13:51,713 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa654a6523c438a866a4a20e0bc6793, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733595228155 2024-12-07T18:13:51,713 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac2914f7d1b948db8c1a6f32eca099dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733595230354 2024-12-07T18:13:51,725 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:51,726 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/7199c87f039a446f9c665a2e6f479a11 is 50, key is test_row_0/C:col10/1733595230354/Put/seqid=0 2024-12-07T18:13:51,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741879_1055 (size=12595) 2024-12-07T18:13:51,743 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/7199c87f039a446f9c665a2e6f479a11 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/7199c87f039a446f9c665a2e6f479a11 2024-12-07T18:13:51,753 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into 7199c87f039a446f9c665a2e6f479a11(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:51,753 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:51,753 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595231633; duration=0sec 2024-12-07T18:13:51,753 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:51,753 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:13:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:51,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:51,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595291934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595291936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595231928, max timestamp=1733595231936Num uniques 859; 2024-12-07T18:13:51,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:13:51,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595231928, max timestamp=1733595231936Num uniques 859; 2024-12-07T18:13:51,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:13:51,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595231928, max timestamp=1733595231936Num uniques 859; 2024-12-07T18:13:51,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:13:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:51,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595291995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:51,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:51,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595291997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T18:13:52,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595292049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595292049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,077 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/d4db4f24a6c84866b644bac684709dde as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/d4db4f24a6c84866b644bac684709dde 2024-12-07T18:13:52,082 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/eb6be9b3eb9d40a88f0e339534078613 2024-12-07T18:13:52,091 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into d4db4f24a6c84866b644bac684709dde(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:52,091 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:52,091 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595231633; duration=0sec 2024-12-07T18:13:52,091 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:52,091 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:13:52,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/63115a777581466fa42fbb2c0ef75787 is 50, key is test_row_0/B:col10/1733595230732/Put/seqid=0 2024-12-07T18:13:52,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741880_1056 (size=12151) 2024-12-07T18:13:52,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595292108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595292109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595292258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595292258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595292319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595292319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,513 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/63115a777581466fa42fbb2c0ef75787 2024-12-07T18:13:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/db73134c671d493abc532e2fd786741c is 50, key is test_row_0/C:col10/1733595230732/Put/seqid=0 2024-12-07T18:13:52,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741881_1057 (size=12151) 2024-12-07T18:13:52,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595292568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595292568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595292628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595292628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:52,931 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/db73134c671d493abc532e2fd786741c 2024-12-07T18:13:52,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/eb6be9b3eb9d40a88f0e339534078613 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/eb6be9b3eb9d40a88f0e339534078613 2024-12-07T18:13:52,946 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/eb6be9b3eb9d40a88f0e339534078613, entries=150, sequenceid=205, filesize=11.9 K 2024-12-07T18:13:52,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/63115a777581466fa42fbb2c0ef75787 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/63115a777581466fa42fbb2c0ef75787 2024-12-07T18:13:52,954 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/63115a777581466fa42fbb2c0ef75787, entries=150, sequenceid=205, filesize=11.9 K 2024-12-07T18:13:52,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/db73134c671d493abc532e2fd786741c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/db73134c671d493abc532e2fd786741c 2024-12-07T18:13:52,961 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/db73134c671d493abc532e2fd786741c, entries=150, sequenceid=205, filesize=11.9 K 2024-12-07T18:13:52,962 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 66753c009d64518d3f4dfefd580afd72 in 1315ms, sequenceid=205, compaction requested=false 2024-12-07T18:13:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:52,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-07T18:13:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-07T18:13:52,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-07T18:13:52,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0910 sec 2024-12-07T18:13:52,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.0990 sec 2024-12-07T18:13:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-07T18:13:53,019 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:13:53,019 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:13:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-07T18:13:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T18:13:53,022 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:13:53,023 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:13:53,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:13:53,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:53,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-07T18:13:53,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:53,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:53,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:53,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:53,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:53,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:53,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/48c508e434404762ab9009a8d93b1612 is 50, key is test_row_0/A:col10/1733595231974/Put/seqid=0 2024-12-07T18:13:53,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741882_1058 (size=14541) 2024-12-07T18:13:53,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/48c508e434404762ab9009a8d93b1612 2024-12-07T18:13:53,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/ceb9edb5232447f19ad0efdc2223d396 is 50, key is test_row_0/B:col10/1733595231974/Put/seqid=0 2024-12-07T18:13:53,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595293097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595293108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741883_1059 (size=12151) 2024-12-07T18:13:53,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/ceb9edb5232447f19ad0efdc2223d396 2024-12-07T18:13:53,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/93a8adb0628947d7978661bef0a3d1bc is 50, key is test_row_0/C:col10/1733595231974/Put/seqid=0 2024-12-07T18:13:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T18:13:53,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741884_1060 (size=12151) 2024-12-07T18:13:53,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/93a8adb0628947d7978661bef0a3d1bc 2024-12-07T18:13:53,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595293139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595293139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/48c508e434404762ab9009a8d93b1612 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/48c508e434404762ab9009a8d93b1612 2024-12-07T18:13:53,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/48c508e434404762ab9009a8d93b1612, entries=200, sequenceid=245, filesize=14.2 K 2024-12-07T18:13:53,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/ceb9edb5232447f19ad0efdc2223d396 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ceb9edb5232447f19ad0efdc2223d396 2024-12-07T18:13:53,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ceb9edb5232447f19ad0efdc2223d396, entries=150, sequenceid=245, filesize=11.9 K 2024-12-07T18:13:53,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/93a8adb0628947d7978661bef0a3d1bc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/93a8adb0628947d7978661bef0a3d1bc 2024-12-07T18:13:53,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/93a8adb0628947d7978661bef0a3d1bc, entries=150, sequenceid=245, filesize=11.9 K 2024-12-07T18:13:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 66753c009d64518d3f4dfefd580afd72 in 99ms, sequenceid=245, compaction requested=true 2024-12-07T18:13:53,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:53,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:13:53,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:53,180 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:53,180 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:53,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:13:53,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:53,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:13:53,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:53,181 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:53,181 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:13:53,181 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,182 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6d2f8ed2a17948e6a0e7ed90d47c334c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/eb6be9b3eb9d40a88f0e339534078613, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/48c508e434404762ab9009a8d93b1612] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=38.4 K 2024-12-07T18:13:53,182 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:53,182 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:13:53,182 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,182 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/d4db4f24a6c84866b644bac684709dde, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/63115a777581466fa42fbb2c0ef75787, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ceb9edb5232447f19ad0efdc2223d396] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=36.0 K 2024-12-07T18:13:53,183 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d2f8ed2a17948e6a0e7ed90d47c334c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733595230354 2024-12-07T18:13:53,184 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting d4db4f24a6c84866b644bac684709dde, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733595230354 2024-12-07T18:13:53,185 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb6be9b3eb9d40a88f0e339534078613, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733595230694 2024-12-07T18:13:53,185 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 63115a777581466fa42fbb2c0ef75787, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733595230694 2024-12-07T18:13:53,186 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ceb9edb5232447f19ad0efdc2223d396, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733595231974 2024-12-07T18:13:53,186 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48c508e434404762ab9009a8d93b1612, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733595231974 2024-12-07T18:13:53,199 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:53,199 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/6debc689be7746d3ad6b412f86f462e9 is 50, key is test_row_0/B:col10/1733595231974/Put/seqid=0 2024-12-07T18:13:53,200 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:53,201 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/60758aa08f264766ac35c08a17c3ce07 is 50, key is test_row_0/A:col10/1733595231974/Put/seqid=0 2024-12-07T18:13:53,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741885_1061 (size=12697) 2024-12-07T18:13:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:53,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T18:13:53,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:53,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:53,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:53,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:53,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:53,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:53,240 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/60758aa08f264766ac35c08a17c3ce07 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60758aa08f264766ac35c08a17c3ce07 2024-12-07T18:13:53,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/e79ddbf1027345968b552486b0ea3482 is 50, key is test_row_0/A:col10/1733595233221/Put/seqid=0 2024-12-07T18:13:53,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741886_1062 (size=12697) 2024-12-07T18:13:53,256 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into 60758aa08f264766ac35c08a17c3ce07(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:53,256 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:53,256 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595233179; duration=0sec 2024-12-07T18:13:53,256 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:53,256 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:13:53,257 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:53,258 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:53,258 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:13:53,258 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,258 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/7199c87f039a446f9c665a2e6f479a11, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/db73134c671d493abc532e2fd786741c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/93a8adb0628947d7978661bef0a3d1bc] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=36.0 K 2024-12-07T18:13:53,261 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7199c87f039a446f9c665a2e6f479a11, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1733595230354 2024-12-07T18:13:53,262 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting db73134c671d493abc532e2fd786741c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733595230694 2024-12-07T18:13:53,262 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93a8adb0628947d7978661bef0a3d1bc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733595231974 2024-12-07T18:13:53,278 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:53,279 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/7eec9b8dd5514d97a105649cdb2d46ab is 50, key is test_row_0/C:col10/1733595231974/Put/seqid=0 2024-12-07T18:13:53,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741887_1063 (size=12147) 2024-12-07T18:13:53,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595293301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595293303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595233276, max timestamp=1733595233300Num uniques 859; 2024-12-07T18:13:53,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595233276, max timestamp=1733595233300Num uniques 859; 2024-12-07T18:13:53,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:53,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:13:53,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595233276, max timestamp=1733595233300Num uniques 859; 2024-12-07T18:13:53,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:53,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741888_1064 (size=12697) 2024-12-07T18:13:53,321 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/7eec9b8dd5514d97a105649cdb2d46ab as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/7eec9b8dd5514d97a105649cdb2d46ab 2024-12-07T18:13:53,330 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into 7eec9b8dd5514d97a105649cdb2d46ab(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:53,330 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:53,330 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595233180; duration=0sec 2024-12-07T18:13:53,331 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:53,331 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:13:53,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:53,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:53,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T18:13:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595293457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595293462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:53,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:53,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595293569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595293569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:53,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:53,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T18:13:53,657 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/6debc689be7746d3ad6b412f86f462e9 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6debc689be7746d3ad6b412f86f462e9 2024-12-07T18:13:53,667 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into 6debc689be7746d3ad6b412f86f462e9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:53,667 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:53,667 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595233180; duration=0sec 2024-12-07T18:13:53,667 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:53,667 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:13:53,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/e79ddbf1027345968b552486b0ea3482 2024-12-07T18:13:53,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/35db0aaeb28d49f380fbda31e14cc531 is 50, key is test_row_0/B:col10/1733595233221/Put/seqid=0 2024-12-07T18:13:53,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741889_1065 (size=9757) 2024-12-07T18:13:53,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595293788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595293788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:53,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:53,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:53,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:53,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:53,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:53,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:54,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:54,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:54,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595294099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595294099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:54,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/35db0aaeb28d49f380fbda31e14cc531 2024-12-07T18:13:54,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/0b997ccf06354c97874ed753fe6a7bb2 is 50, key is test_row_0/C:col10/1733595233221/Put/seqid=0 2024-12-07T18:13:54,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741890_1066 (size=9757) 2024-12-07T18:13:54,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T18:13:54,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595294159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:54,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:54,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595294160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:54,163 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 66753c009d64518d3f4dfefd580afd72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:13:54,164 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 66753c009d64518d3f4dfefd580afd72 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:13:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:54,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:54,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:54,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:54,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/0b997ccf06354c97874ed753fe6a7bb2 2024-12-07T18:13:54,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/e79ddbf1027345968b552486b0ea3482 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/e79ddbf1027345968b552486b0ea3482 2024-12-07T18:13:54,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/e79ddbf1027345968b552486b0ea3482, entries=150, sequenceid=256, filesize=11.9 K 2024-12-07T18:13:54,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/35db0aaeb28d49f380fbda31e14cc531 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/35db0aaeb28d49f380fbda31e14cc531 2024-12-07T18:13:54,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/35db0aaeb28d49f380fbda31e14cc531, entries=100, sequenceid=256, filesize=9.5 K 2024-12-07T18:13:54,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/0b997ccf06354c97874ed753fe6a7bb2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/0b997ccf06354c97874ed753fe6a7bb2 2024-12-07T18:13:54,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/0b997ccf06354c97874ed753fe6a7bb2, entries=100, sequenceid=256, filesize=9.5 K 2024-12-07T18:13:54,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:54,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:54,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 66753c009d64518d3f4dfefd580afd72 in 1347ms, sequenceid=256, compaction requested=false 2024-12-07T18:13:54,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:54,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:54,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-12-07T18:13:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:54,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:54,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:54,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:54,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:54,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/1ca11d576d674f02ac84e519c41a215c is 50, key is test_row_0/A:col10/1733595233460/Put/seqid=0 2024-12-07T18:13:54,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741891_1067 (size=14741) 2024-12-07T18:13:54,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/1ca11d576d674f02ac84e519c41a215c 2024-12-07T18:13:54,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595294643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:54,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595294648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:54,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/2aaeb61ea3b7487ca497f9f615b92627 is 50, key is test_row_0/B:col10/1733595233460/Put/seqid=0 2024-12-07T18:13:54,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741892_1068 (size=12301) 2024-12-07T18:13:54,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/2aaeb61ea3b7487ca497f9f615b92627 2024-12-07T18:13:54,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/61e72748bb9e4b678bd2695208e3ed5b is 50, key is test_row_0/C:col10/1733595233460/Put/seqid=0 2024-12-07T18:13:54,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741893_1069 (size=12301) 2024-12-07T18:13:54,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/61e72748bb9e4b678bd2695208e3ed5b 2024-12-07T18:13:54,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/1ca11d576d674f02ac84e519c41a215c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/1ca11d576d674f02ac84e519c41a215c 2024-12-07T18:13:54,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/1ca11d576d674f02ac84e519c41a215c, entries=200, sequenceid=295, filesize=14.4 K 2024-12-07T18:13:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/2aaeb61ea3b7487ca497f9f615b92627 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2aaeb61ea3b7487ca497f9f615b92627 2024-12-07T18:13:54,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2aaeb61ea3b7487ca497f9f615b92627, entries=150, sequenceid=295, filesize=12.0 K 2024-12-07T18:13:54,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/61e72748bb9e4b678bd2695208e3ed5b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/61e72748bb9e4b678bd2695208e3ed5b 2024-12-07T18:13:54,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/61e72748bb9e4b678bd2695208e3ed5b, entries=150, sequenceid=295, filesize=12.0 K 2024-12-07T18:13:54,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for 66753c009d64518d3f4dfefd580afd72 in 100ms, sequenceid=295, compaction requested=true 2024-12-07T18:13:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:13:54,722 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:54,722 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:13:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:13:54,722 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:54,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-07T18:13:54,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,723 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:13:54,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:54,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:54,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:54,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:54,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:54,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:54,726 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:54,726 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:13:54,726 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,727 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6debc689be7746d3ad6b412f86f462e9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/35db0aaeb28d49f380fbda31e14cc531, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2aaeb61ea3b7487ca497f9f615b92627] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=33.9 K 2024-12-07T18:13:54,727 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:54,727 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:13:54,727 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:54,727 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60758aa08f264766ac35c08a17c3ce07, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/e79ddbf1027345968b552486b0ea3482, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/1ca11d576d674f02ac84e519c41a215c] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=38.7 K 2024-12-07T18:13:54,730 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60758aa08f264766ac35c08a17c3ce07, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733595231974 2024-12-07T18:13:54,730 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6debc689be7746d3ad6b412f86f462e9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733595231974 2024-12-07T18:13:54,731 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting e79ddbf1027345968b552486b0ea3482, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733595233094 2024-12-07T18:13:54,731 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ca11d576d674f02ac84e519c41a215c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733595233448 2024-12-07T18:13:54,731 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 35db0aaeb28d49f380fbda31e14cc531, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733595233094 2024-12-07T18:13:54,734 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aaeb61ea3b7487ca497f9f615b92627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733595233448 2024-12-07T18:13:54,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/872eb52217aa4815b92a44881ab375e7 is 50, key is test_row_0/A:col10/1733595234641/Put/seqid=0 2024-12-07T18:13:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:54,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:54,763 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:54,764 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/3610ee79860148d38c673e8e6f090d6f is 50, key is test_row_0/A:col10/1733595233460/Put/seqid=0 2024-12-07T18:13:54,771 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:54,771 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/ae5fca2fea4f4bffa9ff7ef7095ac0cd is 50, key is test_row_0/B:col10/1733595233460/Put/seqid=0 2024-12-07T18:13:54,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741894_1070 (size=12301) 2024-12-07T18:13:54,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741895_1071 (size=12949) 2024-12-07T18:13:54,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741896_1072 (size=12949) 2024-12-07T18:13:54,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595234848, max timestamp=1733595234868Num uniques 859; 2024-12-07T18:13:54,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595234848, max timestamp=1733595234868Num uniques 859; 2024-12-07T18:13:54,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:54,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:54,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733595234848, max timestamp=1733595234868Num uniques 859; 2024-12-07T18:13:54,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:54,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595294927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:54,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595294930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:54,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:55,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595295038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:55,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595295038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:55,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T18:13:55,190 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/872eb52217aa4815b92a44881ab375e7 2024-12-07T18:13:55,214 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/ae5fca2fea4f4bffa9ff7ef7095ac0cd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ae5fca2fea4f4bffa9ff7ef7095ac0cd 2024-12-07T18:13:55,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/eefbc857b2e0470d985a04443af44c2b is 50, key is test_row_0/B:col10/1733595234641/Put/seqid=0 2024-12-07T18:13:55,215 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/3610ee79860148d38c673e8e6f090d6f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/3610ee79860148d38c673e8e6f090d6f 2024-12-07T18:13:55,231 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into 3610ee79860148d38c673e8e6f090d6f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:55,232 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:55,232 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595234722; duration=0sec 2024-12-07T18:13:55,232 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:55,232 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:13:55,232 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:55,234 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:55,234 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:13:55,234 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:55,234 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/7eec9b8dd5514d97a105649cdb2d46ab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/0b997ccf06354c97874ed753fe6a7bb2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/61e72748bb9e4b678bd2695208e3ed5b] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=33.9 K 2024-12-07T18:13:55,235 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eec9b8dd5514d97a105649cdb2d46ab, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733595231974 2024-12-07T18:13:55,235 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into ae5fca2fea4f4bffa9ff7ef7095ac0cd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:55,235 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:55,235 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595234722; duration=0sec 2024-12-07T18:13:55,235 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:55,235 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:13:55,235 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b997ccf06354c97874ed753fe6a7bb2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1733595233094 2024-12-07T18:13:55,236 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61e72748bb9e4b678bd2695208e3ed5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733595233448 2024-12-07T18:13:55,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741897_1073 (size=12301) 2024-12-07T18:13:55,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33384 deadline: 1733595295249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:55,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595295249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:55,253 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:55,252 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 66753c009d64518d3f4dfefd580afd72 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:13:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:55,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595295249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:55,254 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/b5a21c136cb3425fb7d61043cafb054f is 50, key is test_row_0/C:col10/1733595233460/Put/seqid=0 2024-12-07T18:13:55,252 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/eefbc857b2e0470d985a04443af44c2b 2024-12-07T18:13:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:55,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/11ff30e3d7b142049dd6b8487c58c7a3 is 50, key is test_row_0/C:col10/1733595234641/Put/seqid=0 2024-12-07T18:13:55,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741898_1074 (size=12949) 2024-12-07T18:13:55,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741899_1075 (size=12301) 2024-12-07T18:13:55,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595295560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:55,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:55,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595295568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:55,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:55,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:55,685 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/11ff30e3d7b142049dd6b8487c58c7a3 2024-12-07T18:13:55,685 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/b5a21c136cb3425fb7d61043cafb054f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/b5a21c136cb3425fb7d61043cafb054f 2024-12-07T18:13:55,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/872eb52217aa4815b92a44881ab375e7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/872eb52217aa4815b92a44881ab375e7 2024-12-07T18:13:55,699 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into b5a21c136cb3425fb7d61043cafb054f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:55,699 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:55,699 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595234722; duration=0sec 2024-12-07T18:13:55,699 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:55,699 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:13:55,706 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/872eb52217aa4815b92a44881ab375e7, entries=150, sequenceid=305, filesize=12.0 K 2024-12-07T18:13:55,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/eefbc857b2e0470d985a04443af44c2b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/eefbc857b2e0470d985a04443af44c2b 2024-12-07T18:13:55,713 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/eefbc857b2e0470d985a04443af44c2b, entries=150, sequenceid=305, filesize=12.0 K 2024-12-07T18:13:55,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/11ff30e3d7b142049dd6b8487c58c7a3 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/11ff30e3d7b142049dd6b8487c58c7a3 2024-12-07T18:13:55,722 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/11ff30e3d7b142049dd6b8487c58c7a3, entries=150, sequenceid=305, filesize=12.0 K 2024-12-07T18:13:55,723 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 66753c009d64518d3f4dfefd580afd72 in 1000ms, sequenceid=305, compaction requested=false 2024-12-07T18:13:55,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:55,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:55,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-07T18:13:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-07T18:13:55,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-07T18:13:55,729 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7020 sec 2024-12-07T18:13:55,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.7110 sec 2024-12-07T18:13:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:56,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-07T18:13:56,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:56,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T18:13:56,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:56,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T18:13:56,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:56,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T18:13:56,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/60254b41f0074a66ae9530bf34f193e4 is 50, key is test_row_0/A:col10/1733595236083/Put/seqid=0 2024-12-07T18:13:56,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741900_1076 (size=14741) 2024-12-07T18:13:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:56,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595296108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595296110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:56,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/60254b41f0074a66ae9530bf34f193e4 2024-12-07T18:13:56,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:56,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/2109f784b8a94617a429a83a198e56e3 is 50, key is test_row_0/B:col10/1733595236083/Put/seqid=0 2024-12-07T18:13:56,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741901_1077 (size=12301) 2024-12-07T18:13:56,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/2109f784b8a94617a429a83a198e56e3 2024-12-07T18:13:56,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ffdcef8cb02f43399e9d81e067850173 is 50, key is test_row_0/C:col10/1733595236083/Put/seqid=0 2024-12-07T18:13:56,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741902_1078 (size=12301) 2024-12-07T18:13:56,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595296189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:56,195 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 66753c009d64518d3f4dfefd580afd72 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:13:56,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:56,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595296208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:56,218 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 66753c009d64518d3f4dfefd580afd72 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:13:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:56,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595296219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595296219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:56,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:56,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595296428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:56,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595296428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:56,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:56,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ffdcef8cb02f43399e9d81e067850173 2024-12-07T18:13:56,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/60254b41f0074a66ae9530bf34f193e4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60254b41f0074a66ae9530bf34f193e4 2024-12-07T18:13:56,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60254b41f0074a66ae9530bf34f193e4, entries=200, sequenceid=346, filesize=14.4 K 2024-12-07T18:13:56,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/2109f784b8a94617a429a83a198e56e3 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2109f784b8a94617a429a83a198e56e3 2024-12-07T18:13:56,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2109f784b8a94617a429a83a198e56e3, entries=150, sequenceid=346, filesize=12.0 K 2024-12-07T18:13:56,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ffdcef8cb02f43399e9d81e067850173 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ffdcef8cb02f43399e9d81e067850173 2024-12-07T18:13:56,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ffdcef8cb02f43399e9d81e067850173, entries=150, sequenceid=346, filesize=12.0 K 2024-12-07T18:13:56,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 66753c009d64518d3f4dfefd580afd72 in 567ms, sequenceid=346, compaction requested=true 2024-12-07T18:13:56,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:56,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:13:56,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:56,651 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:56,651 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:56,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:13:56,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:56,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:13:56,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:56,653 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:56,653 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:13:56,653 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:56,653 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ae5fca2fea4f4bffa9ff7ef7095ac0cd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/eefbc857b2e0470d985a04443af44c2b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2109f784b8a94617a429a83a198e56e3] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=36.7 K 2024-12-07T18:13:56,654 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:56,654 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:13:56,654 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:56,654 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/3610ee79860148d38c673e8e6f090d6f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/872eb52217aa4815b92a44881ab375e7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60254b41f0074a66ae9530bf34f193e4] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=39.1 K 2024-12-07T18:13:56,656 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ae5fca2fea4f4bffa9ff7ef7095ac0cd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733595233448 2024-12-07T18:13:56,657 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3610ee79860148d38c673e8e6f090d6f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733595233448 2024-12-07T18:13:56,657 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting eefbc857b2e0470d985a04443af44c2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733595234631 2024-12-07T18:13:56,658 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 872eb52217aa4815b92a44881ab375e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733595234631 2024-12-07T18:13:56,658 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 2109f784b8a94617a429a83a198e56e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733595234925 2024-12-07T18:13:56,659 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60254b41f0074a66ae9530bf34f193e4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733595234905 2024-12-07T18:13:56,672 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:56,673 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/bb36f7cea0544eac8e564eb88270df65 is 50, key is test_row_0/B:col10/1733595236083/Put/seqid=0 2024-12-07T18:13:56,674 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:56,675 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/6147769d3d1745b8bcd74c9dd9ea422c is 50, key is test_row_0/A:col10/1733595236083/Put/seqid=0 2024-12-07T18:13:56,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741903_1079 (size=13051) 2024-12-07T18:13:56,701 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/bb36f7cea0544eac8e564eb88270df65 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/bb36f7cea0544eac8e564eb88270df65 2024-12-07T18:13:56,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741904_1080 (size=13051) 2024-12-07T18:13:56,716 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/6147769d3d1745b8bcd74c9dd9ea422c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6147769d3d1745b8bcd74c9dd9ea422c 2024-12-07T18:13:56,717 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into bb36f7cea0544eac8e564eb88270df65(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:56,718 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:56,718 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595236651; duration=0sec 2024-12-07T18:13:56,718 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:56,718 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:13:56,718 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:56,727 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:56,727 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:13:56,728 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:56,728 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/b5a21c136cb3425fb7d61043cafb054f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/11ff30e3d7b142049dd6b8487c58c7a3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ffdcef8cb02f43399e9d81e067850173] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=36.7 K 2024-12-07T18:13:56,731 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into 6147769d3d1745b8bcd74c9dd9ea422c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:56,731 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:56,731 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595236651; duration=0sec 2024-12-07T18:13:56,731 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:56,731 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:13:56,733 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting b5a21c136cb3425fb7d61043cafb054f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733595233448 2024-12-07T18:13:56,734 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 11ff30e3d7b142049dd6b8487c58c7a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733595234631 2024-12-07T18:13:56,736 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdcef8cb02f43399e9d81e067850173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733595234925 2024-12-07T18:13:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:56,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:13:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:56,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:56,766 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#68 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:56,767 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/faeb6350bd144a0890461a6dac6beb66 is 50, key is test_row_0/C:col10/1733595236083/Put/seqid=0 2024-12-07T18:13:56,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/d744e0db5f974a759e089934d4a1e04a is 50, key is test_row_0/A:col10/1733595236741/Put/seqid=0 2024-12-07T18:13:56,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741905_1081 (size=13051) 2024-12-07T18:13:56,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595236769, max timestamp=1733595236809Num uniques 859; 2024-12-07T18:13:56,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:13:56,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595236769, max timestamp=1733595236809Num uniques 859; 2024-12-07T18:13:56,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:13:56,817 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/faeb6350bd144a0890461a6dac6beb66 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/faeb6350bd144a0890461a6dac6beb66 2024-12-07T18:13:56,817 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595236769, max timestamp=1733595236809Num uniques 859; 2024-12-07T18:13:56,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:13:56,828 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into faeb6350bd144a0890461a6dac6beb66(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:56,828 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:56,828 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595236651; duration=0sec 2024-12-07T18:13:56,828 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:56,828 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:13:56,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741906_1082 (size=14741) 2024-12-07T18:13:56,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/d744e0db5f974a759e089934d4a1e04a 2024-12-07T18:13:56,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/074fc6590550499d8eb28ee1814bacf2 is 50, key is test_row_0/B:col10/1733595236741/Put/seqid=0 2024-12-07T18:13:56,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741907_1083 (size=12301) 2024-12-07T18:13:56,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/074fc6590550499d8eb28ee1814bacf2 2024-12-07T18:13:56,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ea8037f768b4471d9605333e04ccd241 is 50, key is test_row_0/C:col10/1733595236741/Put/seqid=0 2024-12-07T18:13:56,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595296881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:56,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741908_1084 (size=12301) 2024-12-07T18:13:56,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:56,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ea8037f768b4471d9605333e04ccd241 2024-12-07T18:13:56,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595296882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:56,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:56,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/d744e0db5f974a759e089934d4a1e04a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d744e0db5f974a759e089934d4a1e04a 2024-12-07T18:13:56,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d744e0db5f974a759e089934d4a1e04a, entries=200, sequenceid=358, filesize=14.4 K 2024-12-07T18:13:56,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/074fc6590550499d8eb28ee1814bacf2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/074fc6590550499d8eb28ee1814bacf2 2024-12-07T18:13:56,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/074fc6590550499d8eb28ee1814bacf2, entries=150, sequenceid=358, filesize=12.0 K 2024-12-07T18:13:56,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ea8037f768b4471d9605333e04ccd241 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ea8037f768b4471d9605333e04ccd241 2024-12-07T18:13:56,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ea8037f768b4471d9605333e04ccd241, entries=150, sequenceid=358, filesize=12.0 K 2024-12-07T18:13:56,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 66753c009d64518d3f4dfefd580afd72 in 166ms, sequenceid=358, compaction requested=false 2024-12-07T18:13:56,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:56,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:56,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-07T18:13:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:56,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:57,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/28279c705ef445b294606d7675cbf4fb is 50, key is test_row_0/A:col10/1733595236989/Put/seqid=0 2024-12-07T18:13:57,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741909_1085 (size=14741) 2024-12-07T18:13:57,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:57,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595297014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:57,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:57,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:57,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595297018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:57,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595297128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:57,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595297128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-07T18:13:57,178 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:13:57,179 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:13:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-07T18:13:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T18:13:57,182 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:13:57,183 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:13:57,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:13:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T18:13:57,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-07T18:13:57,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:57,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:57,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:57,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:57,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:57,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595297339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:57,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595297339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:57,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/28279c705ef445b294606d7675cbf4fb 2024-12-07T18:13:57,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/8647f5e425374418a7e84a62e369aeaf is 50, key is test_row_0/B:col10/1733595236989/Put/seqid=0 2024-12-07T18:13:57,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741910_1086 (size=12301) 2024-12-07T18:13:57,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/8647f5e425374418a7e84a62e369aeaf 2024-12-07T18:13:57,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/29dd1a17988b4857a9f29ba33b82ec74 is 50, key is test_row_0/C:col10/1733595236989/Put/seqid=0 2024-12-07T18:13:57,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741911_1087 (size=12301) 2024-12-07T18:13:57,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/29dd1a17988b4857a9f29ba33b82ec74 2024-12-07T18:13:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/28279c705ef445b294606d7675cbf4fb as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28279c705ef445b294606d7675cbf4fb 2024-12-07T18:13:57,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28279c705ef445b294606d7675cbf4fb, entries=200, sequenceid=395, filesize=14.4 K 2024-12-07T18:13:57,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/8647f5e425374418a7e84a62e369aeaf as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/8647f5e425374418a7e84a62e369aeaf 2024-12-07T18:13:57,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-07T18:13:57,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:57,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:57,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:57,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:57,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/8647f5e425374418a7e84a62e369aeaf, entries=150, sequenceid=395, filesize=12.0 K 2024-12-07T18:13:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:57,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/29dd1a17988b4857a9f29ba33b82ec74 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/29dd1a17988b4857a9f29ba33b82ec74 2024-12-07T18:13:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T18:13:57,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/29dd1a17988b4857a9f29ba33b82ec74, entries=150, sequenceid=395, filesize=12.0 K 2024-12-07T18:13:57,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 66753c009d64518d3f4dfefd580afd72 in 514ms, sequenceid=395, compaction requested=true 2024-12-07T18:13:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:13:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:57,505 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:13:57,505 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:57,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:57,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:13:57,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:57,507 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:57,507 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:13:57,507 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:57,507 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6147769d3d1745b8bcd74c9dd9ea422c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d744e0db5f974a759e089934d4a1e04a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28279c705ef445b294606d7675cbf4fb] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=41.5 K 2024-12-07T18:13:57,508 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:57,508 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:13:57,508 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6147769d3d1745b8bcd74c9dd9ea422c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733595234925 2024-12-07T18:13:57,508 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:57,508 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/bb36f7cea0544eac8e564eb88270df65, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/074fc6590550499d8eb28ee1814bacf2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/8647f5e425374418a7e84a62e369aeaf] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=36.8 K 2024-12-07T18:13:57,509 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting d744e0db5f974a759e089934d4a1e04a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733595236099 2024-12-07T18:13:57,509 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting bb36f7cea0544eac8e564eb88270df65, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733595234925 2024-12-07T18:13:57,510 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 074fc6590550499d8eb28ee1814bacf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733595236099 2024-12-07T18:13:57,510 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28279c705ef445b294606d7675cbf4fb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733595236856 2024-12-07T18:13:57,511 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 8647f5e425374418a7e84a62e369aeaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733595236878 2024-12-07T18:13:57,524 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:57,525 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/7d3776d8bdfa4c0cbc7823e3b0ca26f9 is 50, key is test_row_0/A:col10/1733595236989/Put/seqid=0 2024-12-07T18:13:57,529 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:57,529 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/e5b0837f07a2403798480598fcd55867 is 50, key is test_row_0/B:col10/1733595236989/Put/seqid=0 2024-12-07T18:13:57,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741912_1088 (size=13153) 2024-12-07T18:13:57,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741913_1089 (size=13153) 2024-12-07T18:13:57,576 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/e5b0837f07a2403798480598fcd55867 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e5b0837f07a2403798480598fcd55867 2024-12-07T18:13:57,586 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into e5b0837f07a2403798480598fcd55867(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:57,586 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:57,586 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595237505; duration=0sec 2024-12-07T18:13:57,586 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:57,586 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:13:57,586 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:57,588 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:57,588 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:13:57,588 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:57,588 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/faeb6350bd144a0890461a6dac6beb66, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ea8037f768b4471d9605333e04ccd241, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/29dd1a17988b4857a9f29ba33b82ec74] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=36.8 K 2024-12-07T18:13:57,589 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting faeb6350bd144a0890461a6dac6beb66, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1733595234925 2024-12-07T18:13:57,589 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8037f768b4471d9605333e04ccd241, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1733595236099 2024-12-07T18:13:57,590 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 29dd1a17988b4857a9f29ba33b82ec74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733595236878 2024-12-07T18:13:57,612 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#77 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:57,613 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/e80d1cce33064d58b66888c9c947c453 is 50, key is test_row_0/C:col10/1733595236989/Put/seqid=0 2024-12-07T18:13:57,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741914_1090 (size=13153) 2024-12-07T18:13:57,644 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/e80d1cce33064d58b66888c9c947c453 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/e80d1cce33064d58b66888c9c947c453 2024-12-07T18:13:57,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-07T18:13:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:57,646 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-07T18:13:57,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:57,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:57,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:57,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:57,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:57,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:57,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/57fb49f7ff084b2b9a5746314ccfe9da is 50, key is test_row_0/A:col10/1733595237010/Put/seqid=0 2024-12-07T18:13:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:57,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:57,668 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into e80d1cce33064d58b66888c9c947c453(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:57,668 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:57,668 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595237506; duration=0sec 2024-12-07T18:13:57,668 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:57,668 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:13:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741915_1091 (size=12301) 2024-12-07T18:13:57,690 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/57fb49f7ff084b2b9a5746314ccfe9da 2024-12-07T18:13:57,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/502b1cdf23a54a63b1bb92aa283c2754 is 50, key is test_row_0/B:col10/1733595237010/Put/seqid=0 2024-12-07T18:13:57,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741916_1092 (size=12301) 2024-12-07T18:13:57,719 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/502b1cdf23a54a63b1bb92aa283c2754 2024-12-07T18:13:57,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595237718, max timestamp=1733595237730Num uniques 859; 2024-12-07T18:13:57,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:57,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595237718, max timestamp=1733595237730Num uniques 859; 2024-12-07T18:13:57,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595237718, max timestamp=1733595237730Num uniques 859; 2024-12-07T18:13:57,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:57,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:57,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/616f5b61a7fd48fc8a793152ef47d494 is 50, key is test_row_0/C:col10/1733595237010/Put/seqid=0 2024-12-07T18:13:57,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741917_1093 (size=12301) 2024-12-07T18:13:57,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595297790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:57,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595297798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:57,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T18:13:57,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595297908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:57,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595297908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:57,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:57,980 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/7d3776d8bdfa4c0cbc7823e3b0ca26f9 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/7d3776d8bdfa4c0cbc7823e3b0ca26f9 2024-12-07T18:13:57,989 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into 7d3776d8bdfa4c0cbc7823e3b0ca26f9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:57,989 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:57,989 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595237505; duration=0sec 2024-12-07T18:13:57,989 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:57,989 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:13:58,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595298129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:58,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595298129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:58,147 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/616f5b61a7fd48fc8a793152ef47d494 2024-12-07T18:13:58,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/57fb49f7ff084b2b9a5746314ccfe9da as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/57fb49f7ff084b2b9a5746314ccfe9da 2024-12-07T18:13:58,159 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/57fb49f7ff084b2b9a5746314ccfe9da, entries=150, sequenceid=405, filesize=12.0 K 2024-12-07T18:13:58,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/502b1cdf23a54a63b1bb92aa283c2754 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/502b1cdf23a54a63b1bb92aa283c2754 2024-12-07T18:13:58,167 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/502b1cdf23a54a63b1bb92aa283c2754, entries=150, sequenceid=405, filesize=12.0 K 2024-12-07T18:13:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/616f5b61a7fd48fc8a793152ef47d494 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/616f5b61a7fd48fc8a793152ef47d494 2024-12-07T18:13:58,173 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/616f5b61a7fd48fc8a793152ef47d494, entries=150, sequenceid=405, filesize=12.0 K 2024-12-07T18:13:58,174 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 66753c009d64518d3f4dfefd580afd72 in 528ms, sequenceid=405, compaction requested=false 2024-12-07T18:13:58,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:58,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:58,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-07T18:13:58,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-07T18:13:58,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-07T18:13:58,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 993 msec 2024-12-07T18:13:58,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 999 msec 2024-12-07T18:13:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-07T18:13:58,319 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:13:58,319 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:13:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-07T18:13:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T18:13:58,322 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:13:58,323 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:13:58,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:13:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T18:13:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:58,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=154.31 KB heapSize=440.77 KB 2024-12-07T18:13:58,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:58,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T18:13:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T18:13:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-07T18:13:58,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/5c8f14c387db46d09606fe0dbdaaeb39 is 50, key is test_row_0/A:col10/1733595237762/Put/seqid=0 2024-12-07T18:13:58,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741918_1094 (size=17181) 2024-12-07T18:13:58,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595298461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:58,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595298463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:58,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T18:13:58,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:58,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:58,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:58,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595298578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:58,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:58,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595298578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:58,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:58,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T18:13:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:58,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:58,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T18:13:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595298789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595298789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:58,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T18:13:58,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:58,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:58,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:58,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/5c8f14c387db46d09606fe0dbdaaeb39 2024-12-07T18:13:58,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/4fda88d542694558b3c88a7b6ded3e7d is 50, key is test_row_0/B:col10/1733595237762/Put/seqid=0 2024-12-07T18:13:58,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741919_1095 (size=12301) 2024-12-07T18:13:58,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/4fda88d542694558b3c88a7b6ded3e7d 2024-12-07T18:13:58,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/1939102e752a46f4acadd83036b73a1e is 50, key is test_row_0/C:col10/1733595237762/Put/seqid=0 2024-12-07T18:13:58,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T18:13:58,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:58,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:58,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:58,945 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T18:13:58,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741920_1096 (size=12301) 2024-12-07T18:13:59,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T18:13:59,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:59,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:59,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:59,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:59,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:59,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:59,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:59,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595299099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:59,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595299099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:59,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T18:13:59,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:59,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:59,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:59,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:59,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:13:59,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/1939102e752a46f4acadd83036b73a1e 2024-12-07T18:13:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/5c8f14c387db46d09606fe0dbdaaeb39 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/5c8f14c387db46d09606fe0dbdaaeb39 2024-12-07T18:13:59,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/5c8f14c387db46d09606fe0dbdaaeb39, entries=250, sequenceid=446, filesize=16.8 K 2024-12-07T18:13:59,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/4fda88d542694558b3c88a7b6ded3e7d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/4fda88d542694558b3c88a7b6ded3e7d 2024-12-07T18:13:59,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/4fda88d542694558b3c88a7b6ded3e7d, entries=150, sequenceid=446, filesize=12.0 K 2024-12-07T18:13:59,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/1939102e752a46f4acadd83036b73a1e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1939102e752a46f4acadd83036b73a1e 2024-12-07T18:13:59,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1939102e752a46f4acadd83036b73a1e, entries=150, sequenceid=446, filesize=12.0 K 2024-12-07T18:13:59,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=26.84 KB/27480 for 66753c009d64518d3f4dfefd580afd72 in 943ms, sequenceid=446, compaction requested=true 2024-12-07T18:13:59,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:59,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:13:59,387 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:59,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:59,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:13:59,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:59,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:13:59,387 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:59,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:59,389 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:59,389 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:13:59,389 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:59,389 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:59,389 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:13:59,389 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/7d3776d8bdfa4c0cbc7823e3b0ca26f9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/57fb49f7ff084b2b9a5746314ccfe9da, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/5c8f14c387db46d09606fe0dbdaaeb39] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=41.6 K 2024-12-07T18:13:59,389 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:59,389 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e5b0837f07a2403798480598fcd55867, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/502b1cdf23a54a63b1bb92aa283c2754, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/4fda88d542694558b3c88a7b6ded3e7d] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=36.9 K 2024-12-07T18:13:59,389 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d3776d8bdfa4c0cbc7823e3b0ca26f9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733595236878 2024-12-07T18:13:59,390 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e5b0837f07a2403798480598fcd55867, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733595236878 2024-12-07T18:13:59,390 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57fb49f7ff084b2b9a5746314ccfe9da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733595237002 2024-12-07T18:13:59,390 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 502b1cdf23a54a63b1bb92aa283c2754, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733595237002 2024-12-07T18:13:59,391 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fda88d542694558b3c88a7b6ded3e7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733595237762 2024-12-07T18:13:59,391 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c8f14c387db46d09606fe0dbdaaeb39, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733595237762 2024-12-07T18:13:59,401 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:59,401 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/494403d086c042d1a6de6b46fb0d411a is 50, key is test_row_0/A:col10/1733595237762/Put/seqid=0 2024-12-07T18:13:59,404 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:59,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-07T18:13:59,406 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/564f7dbc29d54feab8eebbac898082fc is 50, key is test_row_0/B:col10/1733595237762/Put/seqid=0 2024-12-07T18:13:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:59,407 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-07T18:13:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:59,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:13:59,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/b0a3903b53ee410daebf3b77cd02032c is 50, key is test_row_0/A:col10/1733595238453/Put/seqid=0 2024-12-07T18:13:59,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741921_1097 (size=13255) 2024-12-07T18:13:59,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741922_1098 (size=13255) 2024-12-07T18:13:59,436 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/564f7dbc29d54feab8eebbac898082fc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/564f7dbc29d54feab8eebbac898082fc 2024-12-07T18:13:59,446 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into 564f7dbc29d54feab8eebbac898082fc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:59,446 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:59,446 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595239387; duration=0sec 2024-12-07T18:13:59,446 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:13:59,446 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:13:59,446 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:13:59,448 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:13:59,448 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:13:59,449 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:59,449 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/e80d1cce33064d58b66888c9c947c453, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/616f5b61a7fd48fc8a793152ef47d494, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1939102e752a46f4acadd83036b73a1e] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=36.9 K 2024-12-07T18:13:59,449 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e80d1cce33064d58b66888c9c947c453, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1733595236878 2024-12-07T18:13:59,450 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 616f5b61a7fd48fc8a793152ef47d494, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733595237002 2024-12-07T18:13:59,450 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 1939102e752a46f4acadd83036b73a1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733595237762 2024-12-07T18:13:59,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741923_1099 (size=9857) 2024-12-07T18:13:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T18:13:59,459 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/b0a3903b53ee410daebf3b77cd02032c 2024-12-07T18:13:59,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/be65cc9dc223498f9c870dc966d41456 is 50, key is test_row_0/B:col10/1733595238453/Put/seqid=0 2024-12-07T18:13:59,472 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#87 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:13:59,473 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/72dde9a8a2aa4d93959bcd3e7980f208 is 50, key is test_row_0/C:col10/1733595237762/Put/seqid=0 2024-12-07T18:13:59,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741924_1100 (size=9857) 2024-12-07T18:13:59,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741925_1101 (size=13255) 2024-12-07T18:13:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:59,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:13:59,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595239652, max timestamp=1733595239663Num uniques 859; 2024-12-07T18:13:59,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:59,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595239652, max timestamp=1733595239663Num uniques 859; 2024-12-07T18:13:59,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:59,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595239652, max timestamp=1733595239663Num uniques 859; 2024-12-07T18:13:59,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:13:59,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733595239740, max timestamp=1733595239761Num uniques 1068; 2024-12-07T18:13:59,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:13:59,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733595239740, max timestamp=1733595239761Num uniques 1068; 2024-12-07T18:13:59,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-07T18:13:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595299763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:13:59,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-07T18:13:59,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733595239740, max timestamp=1733595239761Num uniques 1068; 2024-12-07T18:13:59,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-07T18:13:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:13:59,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:13:59,831 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/494403d086c042d1a6de6b46fb0d411a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/494403d086c042d1a6de6b46fb0d411a 2024-12-07T18:13:59,837 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into 494403d086c042d1a6de6b46fb0d411a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:59,837 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:59,837 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595239387; duration=0sec 2024-12-07T18:13:59,838 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:59,838 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:13:59,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733595239828, max timestamp=1733595239851Num uniques 1077; 2024-12-07T18:13:59,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-07T18:13:59,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733595239828, max timestamp=1733595239851Num uniques 1077; 2024-12-07T18:13:59,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-07T18:13:59,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733595239828, max timestamp=1733595239851Num uniques 1077; 2024-12-07T18:13:59,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-12-07T18:13:59,901 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/be65cc9dc223498f9c870dc966d41456 2024-12-07T18:13:59,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/2647221c87aa4513a2b0caa1c58889da is 50, key is test_row_0/C:col10/1733595238453/Put/seqid=0 2024-12-07T18:13:59,922 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/72dde9a8a2aa4d93959bcd3e7980f208 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/72dde9a8a2aa4d93959bcd3e7980f208 2024-12-07T18:13:59,929 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into 72dde9a8a2aa4d93959bcd3e7980f208(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:13:59,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733595239905, max timestamp=1733595239923Num uniques 1036; 2024-12-07T18:13:59,929 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:59,929 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595239387; duration=0sec 2024-12-07T18:13:59,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-07T18:13:59,929 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:13:59,929 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:13:59,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733595239905, max timestamp=1733595239923Num uniques 1036; 2024-12-07T18:13:59,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-07T18:13:59,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733595239905, max timestamp=1733595239923Num uniques 1036; 2024-12-07T18:13:59,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-12-07T18:13:59,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741926_1102 (size=9857) 2024-12-07T18:13:59,939 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/2647221c87aa4513a2b0caa1c58889da 2024-12-07T18:13:59,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/b0a3903b53ee410daebf3b77cd02032c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b0a3903b53ee410daebf3b77cd02032c 2024-12-07T18:13:59,950 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b0a3903b53ee410daebf3b77cd02032c, entries=100, sequenceid=453, filesize=9.6 K 2024-12-07T18:13:59,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/be65cc9dc223498f9c870dc966d41456 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/be65cc9dc223498f9c870dc966d41456 2024-12-07T18:13:59,958 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/be65cc9dc223498f9c870dc966d41456, entries=100, sequenceid=453, filesize=9.6 K 2024-12-07T18:13:59,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/2647221c87aa4513a2b0caa1c58889da as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2647221c87aa4513a2b0caa1c58889da 2024-12-07T18:13:59,965 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2647221c87aa4513a2b0caa1c58889da, entries=100, sequenceid=453, filesize=9.6 K 2024-12-07T18:13:59,966 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=93.93 KB/96180 for 66753c009d64518d3f4dfefd580afd72 in 559ms, sequenceid=453, compaction requested=false 2024-12-07T18:13:59,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:13:59,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:13:59,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-07T18:13:59,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-07T18:13:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:13:59,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=100.63 KB heapSize=278.16 KB 2024-12-07T18:13:59,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:13:59,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:59,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:13:59,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:59,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:13:59,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:13:59,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-07T18:13:59,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6450 sec 2024-12-07T18:13:59,976 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6560 sec 2024-12-07T18:13:59,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/a15f7abdb3d240978000df0dcc798fa7 is 50, key is test_row_0/A:col10/1733595239953/Put/seqid=0 2024-12-07T18:14:00,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741927_1103 (size=17181) 2024-12-07T18:14:00,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/a15f7abdb3d240978000df0dcc798fa7 2024-12-07T18:14:00,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595300012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595300016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/f77fc22dbbdf4dacaf352a11f3ea2ffc is 50, key is test_row_0/B:col10/1733595239953/Put/seqid=0 2024-12-07T18:14:00,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741928_1104 (size=12301) 2024-12-07T18:14:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595300130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595300130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33392 deadline: 1733595300218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,228 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 66753c009d64518d3f4dfefd580afd72 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:00,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33410 deadline: 1733595300258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,262 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 66753c009d64518d3f4dfefd580afd72 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8267 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595300338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595300338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/f77fc22dbbdf4dacaf352a11f3ea2ffc 2024-12-07T18:14:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-07T18:14:00,469 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:00,470 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-07T18:14:00,472 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:00,474 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:00,474 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:00,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/a1143fb3fb7f4051bb40966067f5a44f is 50, key is test_row_0/C:col10/1733595239953/Put/seqid=0 2024-12-07T18:14:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-07T18:14:00,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741929_1105 (size=12301) 2024-12-07T18:14:00,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/a1143fb3fb7f4051bb40966067f5a44f 2024-12-07T18:14:00,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/a15f7abdb3d240978000df0dcc798fa7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/a15f7abdb3d240978000df0dcc798fa7 2024-12-07T18:14:00,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/a15f7abdb3d240978000df0dcc798fa7, entries=250, sequenceid=539, filesize=16.8 K 2024-12-07T18:14:00,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/f77fc22dbbdf4dacaf352a11f3ea2ffc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f77fc22dbbdf4dacaf352a11f3ea2ffc 2024-12-07T18:14:00,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f77fc22dbbdf4dacaf352a11f3ea2ffc, entries=150, sequenceid=539, filesize=12.0 K 2024-12-07T18:14:00,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/a1143fb3fb7f4051bb40966067f5a44f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a1143fb3fb7f4051bb40966067f5a44f 2024-12-07T18:14:00,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a1143fb3fb7f4051bb40966067f5a44f, entries=150, sequenceid=539, filesize=12.0 K 2024-12-07T18:14:00,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~298.03 KB/305184, currentSize=73.80 KB/75570 for 66753c009d64518d3f4dfefd580afd72 in 606ms, sequenceid=539, compaction requested=true 2024-12-07T18:14:00,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:00,577 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:00,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:00,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:00,577 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-07T18:14:00,583 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40293 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:00,583 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:14:00,583 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:00,583 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/494403d086c042d1a6de6b46fb0d411a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b0a3903b53ee410daebf3b77cd02032c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/a15f7abdb3d240978000df0dcc798fa7] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=39.3 K 2024-12-07T18:14:00,588 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:00,588 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 494403d086c042d1a6de6b46fb0d411a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733595237762 2024-12-07T18:14:00,589 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:14:00,589 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:00,589 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/564f7dbc29d54feab8eebbac898082fc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/be65cc9dc223498f9c870dc966d41456, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f77fc22dbbdf4dacaf352a11f3ea2ffc] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=34.6 K 2024-12-07T18:14:00,589 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0a3903b53ee410daebf3b77cd02032c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733595238453 2024-12-07T18:14:00,590 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 564f7dbc29d54feab8eebbac898082fc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733595237762 2024-12-07T18:14:00,590 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting a15f7abdb3d240978000df0dcc798fa7, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733595239948 2024-12-07T18:14:00,590 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting be65cc9dc223498f9c870dc966d41456, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733595238453 2024-12-07T18:14:00,591 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting f77fc22dbbdf4dacaf352a11f3ea2ffc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733595239953 2024-12-07T18:14:00,606 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:00,607 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/368df66833c744359530d1a050413c43 is 50, key is test_row_0/A:col10/1733595239953/Put/seqid=0 2024-12-07T18:14:00,610 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:00,611 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/b1784019227f4114b1a062592b7ed6b6 is 50, key is test_row_0/B:col10/1733595239953/Put/seqid=0 2024-12-07T18:14:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:00,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:00,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-07T18:14:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:00,630 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T18:14:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:14:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:14:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:14:00,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:14:00,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:14:00,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741930_1106 (size=13357) 2024-12-07T18:14:00,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/ed3b2d87768e45ab8a519161c67fc27b is 50, key is test_row_0/A:col10/1733595240010/Put/seqid=0 2024-12-07T18:14:00,684 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/368df66833c744359530d1a050413c43 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/368df66833c744359530d1a050413c43 2024-12-07T18:14:00,698 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into 368df66833c744359530d1a050413c43(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:00,698 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:00,698 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595240576; duration=0sec 2024-12-07T18:14:00,699 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:00,699 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:14:00,699 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:00,701 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:00,701 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:14:00,701 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:00,701 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/72dde9a8a2aa4d93959bcd3e7980f208, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2647221c87aa4513a2b0caa1c58889da, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a1143fb3fb7f4051bb40966067f5a44f] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=34.6 K 2024-12-07T18:14:00,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741931_1107 (size=13357) 2024-12-07T18:14:00,708 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72dde9a8a2aa4d93959bcd3e7980f208, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1733595237762 2024-12-07T18:14:00,708 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2647221c87aa4513a2b0caa1c58889da, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1733595238453 2024-12-07T18:14:00,709 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1143fb3fb7f4051bb40966067f5a44f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733595239953 2024-12-07T18:14:00,718 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/b1784019227f4114b1a062592b7ed6b6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b1784019227f4114b1a062592b7ed6b6 2024-12-07T18:14:00,727 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into b1784019227f4114b1a062592b7ed6b6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:00,727 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:00,727 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595240577; duration=0sec 2024-12-07T18:14:00,728 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:00,728 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:14:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741932_1108 (size=12301) 2024-12-07T18:14:00,731 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:00,732 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ba86794d5b6d48258ca93b817fb7324d is 50, key is test_row_0/C:col10/1733595239953/Put/seqid=0 2024-12-07T18:14:00,733 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/ed3b2d87768e45ab8a519161c67fc27b 2024-12-07T18:14:00,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741933_1109 (size=13357) 2024-12-07T18:14:00,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595300739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595300739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,750 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/ba86794d5b6d48258ca93b817fb7324d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ba86794d5b6d48258ca93b817fb7324d 2024-12-07T18:14:00,759 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into ba86794d5b6d48258ca93b817fb7324d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:00,759 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:00,759 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595240615; duration=0sec 2024-12-07T18:14:00,759 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:00,759 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:14:00,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/012b83fed8864262b5cdc8d9342edd15 is 50, key is test_row_0/B:col10/1733595240010/Put/seqid=0 2024-12-07T18:14:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-07T18:14:00,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741934_1110 (size=12301) 2024-12-07T18:14:00,804 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/012b83fed8864262b5cdc8d9342edd15 2024-12-07T18:14:00,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/196a030b4e704b34aefcc1045ae42e7b is 50, key is test_row_0/C:col10/1733595240010/Put/seqid=0 2024-12-07T18:14:00,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741935_1111 (size=12301) 2024-12-07T18:14:00,834 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=553 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/196a030b4e704b34aefcc1045ae42e7b 2024-12-07T18:14:00,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/ed3b2d87768e45ab8a519161c67fc27b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/ed3b2d87768e45ab8a519161c67fc27b 2024-12-07T18:14:00,849 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/ed3b2d87768e45ab8a519161c67fc27b, entries=150, sequenceid=553, filesize=12.0 K 2024-12-07T18:14:00,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595300849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:00,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595300849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:00,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/012b83fed8864262b5cdc8d9342edd15 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/012b83fed8864262b5cdc8d9342edd15 2024-12-07T18:14:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:00,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:00,857 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/012b83fed8864262b5cdc8d9342edd15, entries=150, sequenceid=553, filesize=12.0 K 2024-12-07T18:14:00,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/196a030b4e704b34aefcc1045ae42e7b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/196a030b4e704b34aefcc1045ae42e7b 2024-12-07T18:14:00,872 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/196a030b4e704b34aefcc1045ae42e7b, entries=150, sequenceid=553, filesize=12.0 K 2024-12-07T18:14:00,873 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 66753c009d64518d3f4dfefd580afd72 in 244ms, sequenceid=553, compaction requested=false 2024-12-07T18:14:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-07T18:14:00,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-07T18:14:00,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-07T18:14:00,878 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 401 msec 2024-12-07T18:14:00,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 409 msec 2024-12-07T18:14:01,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T18:14:01,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:14:01,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:01,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:14:01,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:01,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:14:01,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:01,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/78dd999cd63543b19ff2413fa45f0c5f is 50, key is test_row_0/A:col10/1733595240736/Put/seqid=0 2024-12-07T18:14:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:14:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-07T18:14:01,098 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:01,099 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:01,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-07T18:14:01,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-07T18:14:01,101 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:01,102 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:01,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:01,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:01,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595301118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:01,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:01,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595301120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:01,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:01,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741936_1112 (size=12301) 2024-12-07T18:14:01,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/78dd999cd63543b19ff2413fa45f0c5f 2024-12-07T18:14:01,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/da9d9674d8aa47b9a9dfe627257df496 is 50, key is test_row_0/B:col10/1733595240736/Put/seqid=0 2024-12-07T18:14:01,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741937_1113 (size=12301) 2024-12-07T18:14:01,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-07T18:14:01,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595301240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:01,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-07T18:14:01,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:01,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:14:01,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:01,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:01,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595301239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-07T18:14:01,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:01,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:14:01,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:01,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-07T18:14:01,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595301458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:01,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595301459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:01,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-07T18:14:01,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:01,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:14:01,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:01,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/da9d9674d8aa47b9a9dfe627257df496 2024-12-07T18:14:01,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/74d23ab63a9c44819de118e7bb1dbaab is 50, key is test_row_0/C:col10/1733595240736/Put/seqid=0 2024-12-07T18:14:01,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741938_1114 (size=12301) 2024-12-07T18:14:01,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-07T18:14:01,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:01,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:14:01,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:01,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-07T18:14:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595301770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:01,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595301770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:01,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-07T18:14:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:14:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:01,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:02,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-07T18:14:02,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:02,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:14:02,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:02,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:02,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:02,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=576 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/74d23ab63a9c44819de118e7bb1dbaab 2024-12-07T18:14:02,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/78dd999cd63543b19ff2413fa45f0c5f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/78dd999cd63543b19ff2413fa45f0c5f 2024-12-07T18:14:02,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/78dd999cd63543b19ff2413fa45f0c5f, entries=150, sequenceid=576, filesize=12.0 K 2024-12-07T18:14:02,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/da9d9674d8aa47b9a9dfe627257df496 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/da9d9674d8aa47b9a9dfe627257df496 2024-12-07T18:14:02,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/da9d9674d8aa47b9a9dfe627257df496, entries=150, sequenceid=576, filesize=12.0 K 2024-12-07T18:14:02,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/74d23ab63a9c44819de118e7bb1dbaab as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/74d23ab63a9c44819de118e7bb1dbaab 2024-12-07T18:14:02,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/74d23ab63a9c44819de118e7bb1dbaab, entries=150, sequenceid=576, filesize=12.0 K 2024-12-07T18:14:02,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 66753c009d64518d3f4dfefd580afd72 in 1028ms, sequenceid=576, compaction requested=true 2024-12-07T18:14:02,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:02,093 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:02,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-07T18:14:02,093 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:02,094 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:02,094 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:14:02,095 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:02,095 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/368df66833c744359530d1a050413c43, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/ed3b2d87768e45ab8a519161c67fc27b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/78dd999cd63543b19ff2413fa45f0c5f] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=37.1 K 2024-12-07T18:14:02,095 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:02,095 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:14:02,095 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:02,095 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b1784019227f4114b1a062592b7ed6b6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/012b83fed8864262b5cdc8d9342edd15, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/da9d9674d8aa47b9a9dfe627257df496] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=37.1 K 2024-12-07T18:14:02,096 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting b1784019227f4114b1a062592b7ed6b6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733595239953 2024-12-07T18:14:02,096 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 368df66833c744359530d1a050413c43, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733595239953 2024-12-07T18:14:02,097 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 012b83fed8864262b5cdc8d9342edd15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1733595240008 2024-12-07T18:14:02,097 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed3b2d87768e45ab8a519161c67fc27b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1733595240008 2024-12-07T18:14:02,097 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting da9d9674d8aa47b9a9dfe627257df496, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733595240716 2024-12-07T18:14:02,098 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78dd999cd63543b19ff2413fa45f0c5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733595240716 2024-12-07T18:14:02,110 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#102 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:02,110 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/0e36b98e7af240e68f7f112b5540e3da is 50, key is test_row_0/B:col10/1733595240736/Put/seqid=0 2024-12-07T18:14:02,113 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:02,113 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/417d73c0e7734ea3ad1b439761d6d0e7 is 50, key is test_row_0/A:col10/1733595240736/Put/seqid=0 2024-12-07T18:14:02,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741939_1115 (size=13459) 2024-12-07T18:14:02,155 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/0e36b98e7af240e68f7f112b5540e3da as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/0e36b98e7af240e68f7f112b5540e3da 2024-12-07T18:14:02,163 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into 0e36b98e7af240e68f7f112b5540e3da(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:02,163 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:02,163 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595242093; duration=0sec 2024-12-07T18:14:02,163 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:02,163 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:14:02,164 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:02,165 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:02,165 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:14:02,165 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:02,165 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ba86794d5b6d48258ca93b817fb7324d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/196a030b4e704b34aefcc1045ae42e7b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/74d23ab63a9c44819de118e7bb1dbaab] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=37.1 K 2024-12-07T18:14:02,166 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ba86794d5b6d48258ca93b817fb7324d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=539, earliestPutTs=1733595239953 2024-12-07T18:14:02,167 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 196a030b4e704b34aefcc1045ae42e7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=553, earliestPutTs=1733595240008 2024-12-07T18:14:02,168 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 74d23ab63a9c44819de118e7bb1dbaab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733595240716 2024-12-07T18:14:02,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741940_1116 (size=13459) 2024-12-07T18:14:02,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-07T18:14:02,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:02,192 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T18:14:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:14:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:14:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:14:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:02,198 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:02,199 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/8a576d3608b1473d80a0bafa5f0afbb8 is 50, key is test_row_0/C:col10/1733595240736/Put/seqid=0 2024-12-07T18:14:02,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/682252d54ede4704ad0b2eebdbea49af is 50, key is test_row_0/A:col10/1733595241107/Put/seqid=0 2024-12-07T18:14:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-07T18:14:02,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741941_1117 (size=13459) 2024-12-07T18:14:02,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741942_1118 (size=12301) 2024-12-07T18:14:02,255 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=589 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/682252d54ede4704ad0b2eebdbea49af 2024-12-07T18:14:02,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/ea76a277709346d6bf4a8c50e626aec9 is 50, key is test_row_0/B:col10/1733595241107/Put/seqid=0 2024-12-07T18:14:02,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. as already flushing 2024-12-07T18:14:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:14:02,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741943_1119 (size=12301) 2024-12-07T18:14:02,323 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=589 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/ea76a277709346d6bf4a8c50e626aec9 2024-12-07T18:14:02,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595302337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:02,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595302344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:02,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:02,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/1a3a5e54b0a440b191c165375945cbb6 is 50, key is test_row_0/C:col10/1733595241107/Put/seqid=0 2024-12-07T18:14:02,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741944_1120 (size=12301) 2024-12-07T18:14:02,395 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-07T18:14:02,395 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:02,396 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-07T18:14:02,396 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:02,399 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-07T18:14:02,399 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:02,399 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-07T18:14:02,399 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:02,401 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,401 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,401 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,401 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,401 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,401 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,401 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:02,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:02,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:02,402 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:02,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595302448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:02,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595302459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:02,595 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/417d73c0e7734ea3ad1b439761d6d0e7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/417d73c0e7734ea3ad1b439761d6d0e7 2024-12-07T18:14:02,601 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into 417d73c0e7734ea3ad1b439761d6d0e7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:02,601 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:02,601 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595242093; duration=0sec 2024-12-07T18:14:02,601 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:02,601 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:14:02,657 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/8a576d3608b1473d80a0bafa5f0afbb8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/8a576d3608b1473d80a0bafa5f0afbb8 2024-12-07T18:14:02,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33386 deadline: 1733595302658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:02,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:02,662 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into 8a576d3608b1473d80a0bafa5f0afbb8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:02,662 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:02,662 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595242093; duration=0sec 2024-12-07T18:14:02,662 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:02,662 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:14:02,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33438 deadline: 1733595302668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:02,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:02,761 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=589 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/1a3a5e54b0a440b191c165375945cbb6 2024-12-07T18:14:02,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/682252d54ede4704ad0b2eebdbea49af as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/682252d54ede4704ad0b2eebdbea49af 2024-12-07T18:14:02,771 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/682252d54ede4704ad0b2eebdbea49af, entries=150, sequenceid=589, filesize=12.0 K 2024-12-07T18:14:02,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/ea76a277709346d6bf4a8c50e626aec9 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ea76a277709346d6bf4a8c50e626aec9 2024-12-07T18:14:02,778 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ea76a277709346d6bf4a8c50e626aec9, entries=150, sequenceid=589, filesize=12.0 K 2024-12-07T18:14:02,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/1a3a5e54b0a440b191c165375945cbb6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1a3a5e54b0a440b191c165375945cbb6 2024-12-07T18:14:02,783 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1a3a5e54b0a440b191c165375945cbb6, entries=150, sequenceid=589, filesize=12.0 K 2024-12-07T18:14:02,784 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 66753c009d64518d3f4dfefd580afd72 in 593ms, sequenceid=589, compaction requested=false 2024-12-07T18:14:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-07T18:14:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-07T18:14:02,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-07T18:14:02,787 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6840 sec 2024-12-07T18:14:02,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6890 sec 2024-12-07T18:14:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:14:02,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-07T18:14:02,971 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-07T18:14:02,971 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:02,971 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:14:02,971 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:02,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:14:02,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:02,972 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:02,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:14:02,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:02,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/6e94c5ee259a4f5797a62108c7d95236 is 50, key is test_row_0/A:col10/1733595242969/Put/seqid=0 2024-12-07T18:14:02,979 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-07T18:14:02,980 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:02,980 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,980 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:02,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:02,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741945_1121 (size=12301) 2024-12-07T18:14:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-07T18:14:03,250 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:03,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/6e94c5ee259a4f5797a62108c7d95236 2024-12-07T18:14:03,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/b20b5f2f6c1c4011b1b9425d9df87b5f is 50, key is test_row_0/B:col10/1733595242969/Put/seqid=0 2024-12-07T18:14:03,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741946_1122 (size=12301) 2024-12-07T18:14:03,803 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/b20b5f2f6c1c4011b1b9425d9df87b5f 2024-12-07T18:14:03,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/f952472542b74ef2a8d4fc63243864d6 is 50, key is test_row_0/C:col10/1733595242969/Put/seqid=0 2024-12-07T18:14:03,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741947_1123 (size=12301) 2024-12-07T18:14:04,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/f952472542b74ef2a8d4fc63243864d6 2024-12-07T18:14:04,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/6e94c5ee259a4f5797a62108c7d95236 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6e94c5ee259a4f5797a62108c7d95236 2024-12-07T18:14:04,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6e94c5ee259a4f5797a62108c7d95236, entries=150, sequenceid=612, filesize=12.0 K 2024-12-07T18:14:04,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/b20b5f2f6c1c4011b1b9425d9df87b5f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b20b5f2f6c1c4011b1b9425d9df87b5f 2024-12-07T18:14:04,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b20b5f2f6c1c4011b1b9425d9df87b5f, entries=150, sequenceid=612, filesize=12.0 K 2024-12-07T18:14:04,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/f952472542b74ef2a8d4fc63243864d6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/f952472542b74ef2a8d4fc63243864d6 2024-12-07T18:14:04,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/f952472542b74ef2a8d4fc63243864d6, entries=150, sequenceid=612, filesize=12.0 K 2024-12-07T18:14:04,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=6.71 KB/6870 for 66753c009d64518d3f4dfefd580afd72 in 1263ms, sequenceid=612, compaction requested=true 2024-12-07T18:14:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:04,235 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:04,235 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 66753c009d64518d3f4dfefd580afd72:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:04,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:04,236 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:04,236 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:04,236 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/B is initiating minor compaction (all files) 2024-12-07T18:14:04,236 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/A is initiating minor compaction (all files) 2024-12-07T18:14:04,236 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/B in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:04,236 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/A in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:04,236 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/0e36b98e7af240e68f7f112b5540e3da, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ea76a277709346d6bf4a8c50e626aec9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b20b5f2f6c1c4011b1b9425d9df87b5f] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=37.2 K 2024-12-07T18:14:04,236 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/417d73c0e7734ea3ad1b439761d6d0e7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/682252d54ede4704ad0b2eebdbea49af, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6e94c5ee259a4f5797a62108c7d95236] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=37.2 K 2024-12-07T18:14:04,236 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e36b98e7af240e68f7f112b5540e3da, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733595240716 2024-12-07T18:14:04,236 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417d73c0e7734ea3ad1b439761d6d0e7, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733595240716 2024-12-07T18:14:04,237 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ea76a277709346d6bf4a8c50e626aec9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=589, earliestPutTs=1733595241099 2024-12-07T18:14:04,237 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 682252d54ede4704ad0b2eebdbea49af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=589, earliestPutTs=1733595241099 2024-12-07T18:14:04,237 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting b20b5f2f6c1c4011b1b9425d9df87b5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1733595242334 2024-12-07T18:14:04,237 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e94c5ee259a4f5797a62108c7d95236, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1733595242334 2024-12-07T18:14:04,246 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:04,246 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/6aa366aea0f94fb680397fbb8a3b4f7d is 50, key is test_row_0/B:col10/1733595242969/Put/seqid=0 2024-12-07T18:14:04,248 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#A#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:04,248 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/b8446e2dcbf645b59ae331c3112f676c is 50, key is test_row_0/A:col10/1733595242969/Put/seqid=0 2024-12-07T18:14:04,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741948_1124 (size=13561) 2024-12-07T18:14:04,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741949_1125 (size=13561) 2024-12-07T18:14:04,658 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/b8446e2dcbf645b59ae331c3112f676c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b8446e2dcbf645b59ae331c3112f676c 2024-12-07T18:14:04,658 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/6aa366aea0f94fb680397fbb8a3b4f7d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6aa366aea0f94fb680397fbb8a3b4f7d 2024-12-07T18:14:04,664 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/A of 66753c009d64518d3f4dfefd580afd72 into b8446e2dcbf645b59ae331c3112f676c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:04,664 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/B of 66753c009d64518d3f4dfefd580afd72 into 6aa366aea0f94fb680397fbb8a3b4f7d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:04,664 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:04,664 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:04,664 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/A, priority=13, startTime=1733595244235; duration=0sec 2024-12-07T18:14:04,664 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/B, priority=13, startTime=1733595244235; duration=0sec 2024-12-07T18:14:04,664 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:04,664 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:A 2024-12-07T18:14:04,664 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:04,664 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:B 2024-12-07T18:14:04,664 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:04,665 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:04,665 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 66753c009d64518d3f4dfefd580afd72/C is initiating minor compaction (all files) 2024-12-07T18:14:04,665 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 66753c009d64518d3f4dfefd580afd72/C in TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:04,666 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/8a576d3608b1473d80a0bafa5f0afbb8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1a3a5e54b0a440b191c165375945cbb6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/f952472542b74ef2a8d4fc63243864d6] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp, totalSize=37.2 K 2024-12-07T18:14:04,666 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a576d3608b1473d80a0bafa5f0afbb8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=576, earliestPutTs=1733595240716 2024-12-07T18:14:04,666 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a3a5e54b0a440b191c165375945cbb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=589, earliestPutTs=1733595241099 2024-12-07T18:14:04,666 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting f952472542b74ef2a8d4fc63243864d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1733595242334 2024-12-07T18:14:04,674 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 66753c009d64518d3f4dfefd580afd72#C#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:04,675 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/d85ceda1ad36433aa9235f987d5c3bf0 is 50, key is test_row_0/C:col10/1733595242969/Put/seqid=0 2024-12-07T18:14:04,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741950_1126 (size=13561) 2024-12-07T18:14:05,085 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/d85ceda1ad36433aa9235f987d5c3bf0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/d85ceda1ad36433aa9235f987d5c3bf0 2024-12-07T18:14:05,090 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 66753c009d64518d3f4dfefd580afd72/C of 66753c009d64518d3f4dfefd580afd72 into d85ceda1ad36433aa9235f987d5c3bf0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:05,090 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 66753c009d64518d3f4dfefd580afd72: 2024-12-07T18:14:05,090 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., storeName=66753c009d64518d3f4dfefd580afd72/C, priority=13, startTime=1733595244235; duration=0sec 2024-12-07T18:14:05,090 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:05,090 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 66753c009d64518d3f4dfefd580afd72:C 2024-12-07T18:14:05,340 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-07T18:14:05,340 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:05,340 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:05,340 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:05,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:05,434 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T18:14:10,290 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-07T18:14:10,290 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:10,290 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:10,290 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:10,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:10,310 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-07T18:14:10,310 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:10,310 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:10,310 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:10,310 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:10,310 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 191 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 186 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2501 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2519 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1110 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3327 rows 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1116 2024-12-07T18:14:10,311 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3343 rows 2024-12-07T18:14:10,311 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:14:10,311 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:14:10,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:10,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:10,312 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:10,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T18:14:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T18:14:10,324 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595250324"}]},"ts":"1733595250324"} 2024-12-07T18:14:10,326 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T18:14:10,326 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T18:14:10,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T18:14:10,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66753c009d64518d3f4dfefd580afd72, UNASSIGN}] 2024-12-07T18:14:10,333 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66753c009d64518d3f4dfefd580afd72, UNASSIGN 2024-12-07T18:14:10,334 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=66753c009d64518d3f4dfefd580afd72, regionState=CLOSING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:10,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=66753c009d64518d3f4dfefd580afd72, UNASSIGN because future has completed 2024-12-07T18:14:10,337 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T18:14:10,337 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:14:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T18:14:10,493 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:14:10,494 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T18:14:10,494 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 66753c009d64518d3f4dfefd580afd72, disabling compactions & flushes 2024-12-07T18:14:10,494 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:10,494 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:10,494 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. after waiting 0 ms 2024-12-07T18:14:10,495 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:10,495 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 66753c009d64518d3f4dfefd580afd72 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-07T18:14:10,495 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=A 2024-12-07T18:14:10,495 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:10,495 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=B 2024-12-07T18:14:10,495 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:10,495 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 66753c009d64518d3f4dfefd580afd72, store=C 2024-12-07T18:14:10,495 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:10,500 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/caee8dc73a054dfd97bcf6242da98dc8 is 50, key is test_row_0/A:col10/1733595250309/Put/seqid=0 2024-12-07T18:14:10,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741951_1127 (size=12301) 2024-12-07T18:14:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T18:14:10,904 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/caee8dc73a054dfd97bcf6242da98dc8 2024-12-07T18:14:10,912 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/0c41a2a144a9455f8d90f8801df8fe0f is 50, key is test_row_0/B:col10/1733595250309/Put/seqid=0 2024-12-07T18:14:10,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741952_1128 (size=12301) 2024-12-07T18:14:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T18:14:11,317 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/0c41a2a144a9455f8d90f8801df8fe0f 2024-12-07T18:14:11,325 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/f67601d116df42f1a69a6e2b2e8fdddd is 50, key is test_row_0/C:col10/1733595250309/Put/seqid=0 2024-12-07T18:14:11,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741953_1129 (size=12301) 2024-12-07T18:14:11,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T18:14:11,730 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/f67601d116df42f1a69a6e2b2e8fdddd 2024-12-07T18:14:11,735 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/A/caee8dc73a054dfd97bcf6242da98dc8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/caee8dc73a054dfd97bcf6242da98dc8 2024-12-07T18:14:11,739 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/caee8dc73a054dfd97bcf6242da98dc8, entries=150, sequenceid=622, filesize=12.0 K 2024-12-07T18:14:11,740 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/B/0c41a2a144a9455f8d90f8801df8fe0f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/0c41a2a144a9455f8d90f8801df8fe0f 2024-12-07T18:14:11,744 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/0c41a2a144a9455f8d90f8801df8fe0f, entries=150, sequenceid=622, filesize=12.0 K 2024-12-07T18:14:11,745 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/.tmp/C/f67601d116df42f1a69a6e2b2e8fdddd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/f67601d116df42f1a69a6e2b2e8fdddd 2024-12-07T18:14:11,748 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/f67601d116df42f1a69a6e2b2e8fdddd, entries=150, sequenceid=622, filesize=12.0 K 2024-12-07T18:14:11,749 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 66753c009d64518d3f4dfefd580afd72 in 1254ms, sequenceid=622, compaction requested=false 2024-12-07T18:14:11,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cfa12dec4f30457a920c7b1e6a53b18e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/728c0ff86fc643b9992e3edf9c1666e8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cb1b3cbef1884ff49abd74ea62371386, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/752ed38067884ab788e1a1633d6b1ae8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/353bc3fbc8a7493baca021ef0e4fb86e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/35546658084743c1bc99600714b481e9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28a2e705c281427fa02b4ae141bc4625, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d6dee20ff4c64c2896695fb8e90485e0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/f1dfad50f4b04a29b13aa5c916ba5300, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b7a2c42d304e4cbb966ff43f124e951a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/83ba726f19c94a56b50221936ccf3f90, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/9e99d9a979124ecb9854314a5e1ac33d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6d2f8ed2a17948e6a0e7ed90d47c334c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/65b1f66679da43189a444ca854ccdba0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/eb6be9b3eb9d40a88f0e339534078613, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/48c508e434404762ab9009a8d93b1612, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60758aa08f264766ac35c08a17c3ce07, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/e79ddbf1027345968b552486b0ea3482, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/1ca11d576d674f02ac84e519c41a215c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/3610ee79860148d38c673e8e6f090d6f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/872eb52217aa4815b92a44881ab375e7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60254b41f0074a66ae9530bf34f193e4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6147769d3d1745b8bcd74c9dd9ea422c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d744e0db5f974a759e089934d4a1e04a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28279c705ef445b294606d7675cbf4fb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/7d3776d8bdfa4c0cbc7823e3b0ca26f9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/57fb49f7ff084b2b9a5746314ccfe9da, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/5c8f14c387db46d09606fe0dbdaaeb39, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/494403d086c042d1a6de6b46fb0d411a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b0a3903b53ee410daebf3b77cd02032c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/a15f7abdb3d240978000df0dcc798fa7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/368df66833c744359530d1a050413c43, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/ed3b2d87768e45ab8a519161c67fc27b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/417d73c0e7734ea3ad1b439761d6d0e7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/78dd999cd63543b19ff2413fa45f0c5f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/682252d54ede4704ad0b2eebdbea49af, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6e94c5ee259a4f5797a62108c7d95236] to archive 2024-12-07T18:14:11,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:14:11,758 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cfa12dec4f30457a920c7b1e6a53b18e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cfa12dec4f30457a920c7b1e6a53b18e 2024-12-07T18:14:11,759 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/728c0ff86fc643b9992e3edf9c1666e8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/728c0ff86fc643b9992e3edf9c1666e8 2024-12-07T18:14:11,761 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cb1b3cbef1884ff49abd74ea62371386 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/cb1b3cbef1884ff49abd74ea62371386 2024-12-07T18:14:11,762 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/752ed38067884ab788e1a1633d6b1ae8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/752ed38067884ab788e1a1633d6b1ae8 2024-12-07T18:14:11,763 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/353bc3fbc8a7493baca021ef0e4fb86e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/353bc3fbc8a7493baca021ef0e4fb86e 2024-12-07T18:14:11,764 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/35546658084743c1bc99600714b481e9 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/35546658084743c1bc99600714b481e9 2024-12-07T18:14:11,766 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28a2e705c281427fa02b4ae141bc4625 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28a2e705c281427fa02b4ae141bc4625 2024-12-07T18:14:11,767 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d6dee20ff4c64c2896695fb8e90485e0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d6dee20ff4c64c2896695fb8e90485e0 2024-12-07T18:14:11,768 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/f1dfad50f4b04a29b13aa5c916ba5300 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/f1dfad50f4b04a29b13aa5c916ba5300 2024-12-07T18:14:11,769 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b7a2c42d304e4cbb966ff43f124e951a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b7a2c42d304e4cbb966ff43f124e951a 2024-12-07T18:14:11,770 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/83ba726f19c94a56b50221936ccf3f90 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/83ba726f19c94a56b50221936ccf3f90 2024-12-07T18:14:11,771 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/9e99d9a979124ecb9854314a5e1ac33d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/9e99d9a979124ecb9854314a5e1ac33d 2024-12-07T18:14:11,773 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6d2f8ed2a17948e6a0e7ed90d47c334c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6d2f8ed2a17948e6a0e7ed90d47c334c 2024-12-07T18:14:11,774 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/65b1f66679da43189a444ca854ccdba0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/65b1f66679da43189a444ca854ccdba0 2024-12-07T18:14:11,775 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/eb6be9b3eb9d40a88f0e339534078613 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/eb6be9b3eb9d40a88f0e339534078613 2024-12-07T18:14:11,776 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/48c508e434404762ab9009a8d93b1612 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/48c508e434404762ab9009a8d93b1612 2024-12-07T18:14:11,777 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60758aa08f264766ac35c08a17c3ce07 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60758aa08f264766ac35c08a17c3ce07 2024-12-07T18:14:11,778 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/e79ddbf1027345968b552486b0ea3482 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/e79ddbf1027345968b552486b0ea3482 2024-12-07T18:14:11,779 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/1ca11d576d674f02ac84e519c41a215c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/1ca11d576d674f02ac84e519c41a215c 2024-12-07T18:14:11,780 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/3610ee79860148d38c673e8e6f090d6f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/3610ee79860148d38c673e8e6f090d6f 2024-12-07T18:14:11,781 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/872eb52217aa4815b92a44881ab375e7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/872eb52217aa4815b92a44881ab375e7 2024-12-07T18:14:11,783 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60254b41f0074a66ae9530bf34f193e4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/60254b41f0074a66ae9530bf34f193e4 2024-12-07T18:14:11,784 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6147769d3d1745b8bcd74c9dd9ea422c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6147769d3d1745b8bcd74c9dd9ea422c 2024-12-07T18:14:11,785 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d744e0db5f974a759e089934d4a1e04a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/d744e0db5f974a759e089934d4a1e04a 2024-12-07T18:14:11,786 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28279c705ef445b294606d7675cbf4fb to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/28279c705ef445b294606d7675cbf4fb 2024-12-07T18:14:11,787 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/7d3776d8bdfa4c0cbc7823e3b0ca26f9 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/7d3776d8bdfa4c0cbc7823e3b0ca26f9 2024-12-07T18:14:11,789 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/57fb49f7ff084b2b9a5746314ccfe9da to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/57fb49f7ff084b2b9a5746314ccfe9da 2024-12-07T18:14:11,790 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/5c8f14c387db46d09606fe0dbdaaeb39 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/5c8f14c387db46d09606fe0dbdaaeb39 2024-12-07T18:14:11,791 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/494403d086c042d1a6de6b46fb0d411a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/494403d086c042d1a6de6b46fb0d411a 2024-12-07T18:14:11,792 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b0a3903b53ee410daebf3b77cd02032c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b0a3903b53ee410daebf3b77cd02032c 2024-12-07T18:14:11,793 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/a15f7abdb3d240978000df0dcc798fa7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/a15f7abdb3d240978000df0dcc798fa7 2024-12-07T18:14:11,795 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/368df66833c744359530d1a050413c43 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/368df66833c744359530d1a050413c43 2024-12-07T18:14:11,796 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/ed3b2d87768e45ab8a519161c67fc27b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/ed3b2d87768e45ab8a519161c67fc27b 2024-12-07T18:14:11,797 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/417d73c0e7734ea3ad1b439761d6d0e7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/417d73c0e7734ea3ad1b439761d6d0e7 2024-12-07T18:14:11,798 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/78dd999cd63543b19ff2413fa45f0c5f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/78dd999cd63543b19ff2413fa45f0c5f 2024-12-07T18:14:11,799 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/682252d54ede4704ad0b2eebdbea49af to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/682252d54ede4704ad0b2eebdbea49af 2024-12-07T18:14:11,801 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6e94c5ee259a4f5797a62108c7d95236 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/6e94c5ee259a4f5797a62108c7d95236 2024-12-07T18:14:11,819 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/7df2c2abb5654b97bbc6540b2e1db4a6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b4c6a44c3f044d55b1203629765f3781, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/97f3fdc69de74e05b28fb99a62fcb98b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6dda319301654397a38ee5f837af986a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/c42e65275d134983b72f6ac1fc13265f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/06a43bf855994b3eacd151248c99fa54, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/590245b75cfd4185b2a061a27d825f82, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e047e9b0853e42be90393854bd7bf3ea, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/400ae14df2a641ddb3b97887e6524bd2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/57eb56cf845d4aa89b3e26fb390ee7dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/3786526879354832b133d4242e18771e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e3406dab37e24e36a72240b052c4570b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/d4db4f24a6c84866b644bac684709dde, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f46fcff3ddb944f3a731a44782333603, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/63115a777581466fa42fbb2c0ef75787, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6debc689be7746d3ad6b412f86f462e9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ceb9edb5232447f19ad0efdc2223d396, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/35db0aaeb28d49f380fbda31e14cc531, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ae5fca2fea4f4bffa9ff7ef7095ac0cd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2aaeb61ea3b7487ca497f9f615b92627, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/eefbc857b2e0470d985a04443af44c2b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/bb36f7cea0544eac8e564eb88270df65, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2109f784b8a94617a429a83a198e56e3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/074fc6590550499d8eb28ee1814bacf2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e5b0837f07a2403798480598fcd55867, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/8647f5e425374418a7e84a62e369aeaf, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/502b1cdf23a54a63b1bb92aa283c2754, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/564f7dbc29d54feab8eebbac898082fc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/4fda88d542694558b3c88a7b6ded3e7d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/be65cc9dc223498f9c870dc966d41456, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b1784019227f4114b1a062592b7ed6b6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f77fc22dbbdf4dacaf352a11f3ea2ffc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/012b83fed8864262b5cdc8d9342edd15, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/0e36b98e7af240e68f7f112b5540e3da, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/da9d9674d8aa47b9a9dfe627257df496, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ea76a277709346d6bf4a8c50e626aec9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b20b5f2f6c1c4011b1b9425d9df87b5f] to archive 2024-12-07T18:14:11,820 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:14:11,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/7df2c2abb5654b97bbc6540b2e1db4a6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/7df2c2abb5654b97bbc6540b2e1db4a6 2024-12-07T18:14:11,824 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b4c6a44c3f044d55b1203629765f3781 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b4c6a44c3f044d55b1203629765f3781 2024-12-07T18:14:11,825 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/97f3fdc69de74e05b28fb99a62fcb98b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/97f3fdc69de74e05b28fb99a62fcb98b 2024-12-07T18:14:11,826 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6dda319301654397a38ee5f837af986a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6dda319301654397a38ee5f837af986a 2024-12-07T18:14:11,827 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/c42e65275d134983b72f6ac1fc13265f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/c42e65275d134983b72f6ac1fc13265f 2024-12-07T18:14:11,828 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/06a43bf855994b3eacd151248c99fa54 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/06a43bf855994b3eacd151248c99fa54 2024-12-07T18:14:11,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/590245b75cfd4185b2a061a27d825f82 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/590245b75cfd4185b2a061a27d825f82 2024-12-07T18:14:11,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e047e9b0853e42be90393854bd7bf3ea to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e047e9b0853e42be90393854bd7bf3ea 2024-12-07T18:14:11,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/400ae14df2a641ddb3b97887e6524bd2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/400ae14df2a641ddb3b97887e6524bd2 2024-12-07T18:14:11,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/57eb56cf845d4aa89b3e26fb390ee7dd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/57eb56cf845d4aa89b3e26fb390ee7dd 2024-12-07T18:14:11,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/3786526879354832b133d4242e18771e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/3786526879354832b133d4242e18771e 2024-12-07T18:14:11,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e3406dab37e24e36a72240b052c4570b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e3406dab37e24e36a72240b052c4570b 2024-12-07T18:14:11,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/d4db4f24a6c84866b644bac684709dde to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/d4db4f24a6c84866b644bac684709dde 2024-12-07T18:14:11,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f46fcff3ddb944f3a731a44782333603 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f46fcff3ddb944f3a731a44782333603 2024-12-07T18:14:11,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/63115a777581466fa42fbb2c0ef75787 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/63115a777581466fa42fbb2c0ef75787 2024-12-07T18:14:11,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6debc689be7746d3ad6b412f86f462e9 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6debc689be7746d3ad6b412f86f462e9 2024-12-07T18:14:11,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ceb9edb5232447f19ad0efdc2223d396 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ceb9edb5232447f19ad0efdc2223d396 2024-12-07T18:14:11,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/35db0aaeb28d49f380fbda31e14cc531 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/35db0aaeb28d49f380fbda31e14cc531 2024-12-07T18:14:11,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ae5fca2fea4f4bffa9ff7ef7095ac0cd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ae5fca2fea4f4bffa9ff7ef7095ac0cd 2024-12-07T18:14:11,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2aaeb61ea3b7487ca497f9f615b92627 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2aaeb61ea3b7487ca497f9f615b92627 2024-12-07T18:14:11,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/eefbc857b2e0470d985a04443af44c2b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/eefbc857b2e0470d985a04443af44c2b 2024-12-07T18:14:11,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/bb36f7cea0544eac8e564eb88270df65 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/bb36f7cea0544eac8e564eb88270df65 2024-12-07T18:14:11,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2109f784b8a94617a429a83a198e56e3 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/2109f784b8a94617a429a83a198e56e3 2024-12-07T18:14:11,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/074fc6590550499d8eb28ee1814bacf2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/074fc6590550499d8eb28ee1814bacf2 2024-12-07T18:14:11,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e5b0837f07a2403798480598fcd55867 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/e5b0837f07a2403798480598fcd55867 2024-12-07T18:14:11,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/8647f5e425374418a7e84a62e369aeaf to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/8647f5e425374418a7e84a62e369aeaf 2024-12-07T18:14:11,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/502b1cdf23a54a63b1bb92aa283c2754 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/502b1cdf23a54a63b1bb92aa283c2754 2024-12-07T18:14:11,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/564f7dbc29d54feab8eebbac898082fc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/564f7dbc29d54feab8eebbac898082fc 2024-12-07T18:14:11,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/4fda88d542694558b3c88a7b6ded3e7d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/4fda88d542694558b3c88a7b6ded3e7d 2024-12-07T18:14:11,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/be65cc9dc223498f9c870dc966d41456 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/be65cc9dc223498f9c870dc966d41456 2024-12-07T18:14:11,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b1784019227f4114b1a062592b7ed6b6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b1784019227f4114b1a062592b7ed6b6 2024-12-07T18:14:11,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f77fc22dbbdf4dacaf352a11f3ea2ffc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/f77fc22dbbdf4dacaf352a11f3ea2ffc 2024-12-07T18:14:11,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/012b83fed8864262b5cdc8d9342edd15 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/012b83fed8864262b5cdc8d9342edd15 2024-12-07T18:14:11,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/0e36b98e7af240e68f7f112b5540e3da to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/0e36b98e7af240e68f7f112b5540e3da 2024-12-07T18:14:11,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/da9d9674d8aa47b9a9dfe627257df496 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/da9d9674d8aa47b9a9dfe627257df496 2024-12-07T18:14:11,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ea76a277709346d6bf4a8c50e626aec9 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/ea76a277709346d6bf4a8c50e626aec9 2024-12-07T18:14:11,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b20b5f2f6c1c4011b1b9425d9df87b5f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/b20b5f2f6c1c4011b1b9425d9df87b5f 2024-12-07T18:14:11,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2110d0c13b8e47c8a4600b1c7d6cebba, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/c841f041d6314887a8246ea8b33dbeb6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/322d0ba18d4a4aa1bf3049d0d34b77ab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a569cedd82c747d88c4c88af141afd1a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ae0ac7964c654fc29cf30f1e1c441de8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/e41922ab38a740ba908ace4dae771150, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/5ab7901088ca439ebfdc47a03aabafe6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/dbcf4e86b0064b668841c8a76a17e4de, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/563da3eaa9d549388a6b4b9f02690af0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a079029c0bbd44fdb6a300ab5b791d62, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/cb1dbe2ca41744d0b528c1e941face92, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2fa654a6523c438a866a4a20e0bc6793, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/7199c87f039a446f9c665a2e6f479a11, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ac2914f7d1b948db8c1a6f32eca099dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/db73134c671d493abc532e2fd786741c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/7eec9b8dd5514d97a105649cdb2d46ab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/93a8adb0628947d7978661bef0a3d1bc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/0b997ccf06354c97874ed753fe6a7bb2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/b5a21c136cb3425fb7d61043cafb054f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/61e72748bb9e4b678bd2695208e3ed5b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/11ff30e3d7b142049dd6b8487c58c7a3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/faeb6350bd144a0890461a6dac6beb66, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ffdcef8cb02f43399e9d81e067850173, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ea8037f768b4471d9605333e04ccd241, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/e80d1cce33064d58b66888c9c947c453, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/29dd1a17988b4857a9f29ba33b82ec74, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/616f5b61a7fd48fc8a793152ef47d494, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/72dde9a8a2aa4d93959bcd3e7980f208, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1939102e752a46f4acadd83036b73a1e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2647221c87aa4513a2b0caa1c58889da, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ba86794d5b6d48258ca93b817fb7324d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a1143fb3fb7f4051bb40966067f5a44f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/196a030b4e704b34aefcc1045ae42e7b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/8a576d3608b1473d80a0bafa5f0afbb8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/74d23ab63a9c44819de118e7bb1dbaab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1a3a5e54b0a440b191c165375945cbb6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/f952472542b74ef2a8d4fc63243864d6] to archive 2024-12-07T18:14:11,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:14:11,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2110d0c13b8e47c8a4600b1c7d6cebba to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2110d0c13b8e47c8a4600b1c7d6cebba 2024-12-07T18:14:11,869 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/c841f041d6314887a8246ea8b33dbeb6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/c841f041d6314887a8246ea8b33dbeb6 2024-12-07T18:14:11,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/322d0ba18d4a4aa1bf3049d0d34b77ab to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/322d0ba18d4a4aa1bf3049d0d34b77ab 2024-12-07T18:14:11,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a569cedd82c747d88c4c88af141afd1a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a569cedd82c747d88c4c88af141afd1a 2024-12-07T18:14:11,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ae0ac7964c654fc29cf30f1e1c441de8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ae0ac7964c654fc29cf30f1e1c441de8 2024-12-07T18:14:11,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/e41922ab38a740ba908ace4dae771150 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/e41922ab38a740ba908ace4dae771150 2024-12-07T18:14:11,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/5ab7901088ca439ebfdc47a03aabafe6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/5ab7901088ca439ebfdc47a03aabafe6 2024-12-07T18:14:11,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/dbcf4e86b0064b668841c8a76a17e4de to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/dbcf4e86b0064b668841c8a76a17e4de 2024-12-07T18:14:11,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/563da3eaa9d549388a6b4b9f02690af0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/563da3eaa9d549388a6b4b9f02690af0 2024-12-07T18:14:11,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a079029c0bbd44fdb6a300ab5b791d62 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a079029c0bbd44fdb6a300ab5b791d62 2024-12-07T18:14:11,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/cb1dbe2ca41744d0b528c1e941face92 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/cb1dbe2ca41744d0b528c1e941face92 2024-12-07T18:14:11,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2fa654a6523c438a866a4a20e0bc6793 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2fa654a6523c438a866a4a20e0bc6793 2024-12-07T18:14:11,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/7199c87f039a446f9c665a2e6f479a11 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/7199c87f039a446f9c665a2e6f479a11 2024-12-07T18:14:11,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ac2914f7d1b948db8c1a6f32eca099dd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ac2914f7d1b948db8c1a6f32eca099dd 2024-12-07T18:14:11,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/db73134c671d493abc532e2fd786741c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/db73134c671d493abc532e2fd786741c 2024-12-07T18:14:11,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/7eec9b8dd5514d97a105649cdb2d46ab to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/7eec9b8dd5514d97a105649cdb2d46ab 2024-12-07T18:14:11,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/93a8adb0628947d7978661bef0a3d1bc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/93a8adb0628947d7978661bef0a3d1bc 2024-12-07T18:14:11,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/0b997ccf06354c97874ed753fe6a7bb2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/0b997ccf06354c97874ed753fe6a7bb2 2024-12-07T18:14:11,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/b5a21c136cb3425fb7d61043cafb054f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/b5a21c136cb3425fb7d61043cafb054f 2024-12-07T18:14:11,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/61e72748bb9e4b678bd2695208e3ed5b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/61e72748bb9e4b678bd2695208e3ed5b 2024-12-07T18:14:11,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/11ff30e3d7b142049dd6b8487c58c7a3 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/11ff30e3d7b142049dd6b8487c58c7a3 2024-12-07T18:14:11,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/faeb6350bd144a0890461a6dac6beb66 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/faeb6350bd144a0890461a6dac6beb66 2024-12-07T18:14:11,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ffdcef8cb02f43399e9d81e067850173 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ffdcef8cb02f43399e9d81e067850173 2024-12-07T18:14:11,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ea8037f768b4471d9605333e04ccd241 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ea8037f768b4471d9605333e04ccd241 2024-12-07T18:14:11,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/e80d1cce33064d58b66888c9c947c453 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/e80d1cce33064d58b66888c9c947c453 2024-12-07T18:14:11,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/29dd1a17988b4857a9f29ba33b82ec74 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/29dd1a17988b4857a9f29ba33b82ec74 2024-12-07T18:14:11,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/616f5b61a7fd48fc8a793152ef47d494 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/616f5b61a7fd48fc8a793152ef47d494 2024-12-07T18:14:11,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/72dde9a8a2aa4d93959bcd3e7980f208 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/72dde9a8a2aa4d93959bcd3e7980f208 2024-12-07T18:14:11,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1939102e752a46f4acadd83036b73a1e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1939102e752a46f4acadd83036b73a1e 2024-12-07T18:14:11,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2647221c87aa4513a2b0caa1c58889da to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/2647221c87aa4513a2b0caa1c58889da 2024-12-07T18:14:11,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ba86794d5b6d48258ca93b817fb7324d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/ba86794d5b6d48258ca93b817fb7324d 2024-12-07T18:14:11,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a1143fb3fb7f4051bb40966067f5a44f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/a1143fb3fb7f4051bb40966067f5a44f 2024-12-07T18:14:11,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/196a030b4e704b34aefcc1045ae42e7b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/196a030b4e704b34aefcc1045ae42e7b 2024-12-07T18:14:11,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/8a576d3608b1473d80a0bafa5f0afbb8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/8a576d3608b1473d80a0bafa5f0afbb8 2024-12-07T18:14:11,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/74d23ab63a9c44819de118e7bb1dbaab to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/74d23ab63a9c44819de118e7bb1dbaab 2024-12-07T18:14:11,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1a3a5e54b0a440b191c165375945cbb6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/1a3a5e54b0a440b191c165375945cbb6 2024-12-07T18:14:11,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/f952472542b74ef2a8d4fc63243864d6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/f952472542b74ef2a8d4fc63243864d6 2024-12-07T18:14:11,915 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/recovered.edits/625.seqid, newMaxSeqId=625, maxSeqId=1 2024-12-07T18:14:11,916 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. 2024-12-07T18:14:11,916 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 66753c009d64518d3f4dfefd580afd72: Waiting for close lock at 1733595250494Running coprocessor pre-close hooks at 1733595250494Disabling compacts and flushes for region at 1733595250494Disabling writes for close at 1733595250494Obtaining lock to block concurrent updates at 1733595250495 (+1 ms)Preparing flush snapshotting stores in 66753c009d64518d3f4dfefd580afd72 at 1733595250495Finished memstore snapshotting TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733595250495Flushing stores of TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72. at 1733595250496 (+1 ms)Flushing 66753c009d64518d3f4dfefd580afd72/A: creating writer at 1733595250496Flushing 66753c009d64518d3f4dfefd580afd72/A: appending metadata at 1733595250499 (+3 ms)Flushing 66753c009d64518d3f4dfefd580afd72/A: closing flushed file at 1733595250499Flushing 66753c009d64518d3f4dfefd580afd72/B: creating writer at 1733595250909 (+410 ms)Flushing 66753c009d64518d3f4dfefd580afd72/B: appending metadata at 1733595250912 (+3 ms)Flushing 66753c009d64518d3f4dfefd580afd72/B: closing flushed file at 1733595250912Flushing 66753c009d64518d3f4dfefd580afd72/C: creating writer at 1733595251322 (+410 ms)Flushing 66753c009d64518d3f4dfefd580afd72/C: appending metadata at 1733595251325 (+3 ms)Flushing 66753c009d64518d3f4dfefd580afd72/C: closing flushed file at 1733595251325Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@239ded49: reopening flushed file at 1733595251735 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@403c6f4: reopening flushed file at 1733595251739 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ea5a202: reopening flushed file at 1733595251744 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 66753c009d64518d3f4dfefd580afd72 in 1254ms, sequenceid=622, compaction requested=false at 1733595251749 (+5 ms)Writing region close event to WAL at 1733595251911 (+162 ms)Running coprocessor post-close hooks at 1733595251915 (+4 ms)Closed at 1733595251916 (+1 ms) 2024-12-07T18:14:11,919 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 66753c009d64518d3f4dfefd580afd72 2024-12-07T18:14:11,919 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=66753c009d64518d3f4dfefd580afd72, regionState=CLOSED 2024-12-07T18:14:11,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:14:11,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-07T18:14:11,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 66753c009d64518d3f4dfefd580afd72, server=92acd7e2389b,33947,1733595217075 in 1.5850 sec 2024-12-07T18:14:11,925 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-07T18:14:11,926 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=66753c009d64518d3f4dfefd580afd72, UNASSIGN in 1.5920 sec 2024-12-07T18:14:11,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-07T18:14:11,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5980 sec 2024-12-07T18:14:11,929 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595251929"}]},"ts":"1733595251929"} 2024-12-07T18:14:11,931 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T18:14:11,931 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T18:14:11,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6140 sec 2024-12-07T18:14:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-07T18:14:12,469 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:12,472 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T18:14:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:12,477 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-07T18:14:12,479 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:12,483 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72 2024-12-07T18:14:12,487 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/recovered.edits] 2024-12-07T18:14:12,490 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b8446e2dcbf645b59ae331c3112f676c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/b8446e2dcbf645b59ae331c3112f676c 2024-12-07T18:14:12,491 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/caee8dc73a054dfd97bcf6242da98dc8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/A/caee8dc73a054dfd97bcf6242da98dc8 2024-12-07T18:14:12,493 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/0c41a2a144a9455f8d90f8801df8fe0f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/0c41a2a144a9455f8d90f8801df8fe0f 2024-12-07T18:14:12,495 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6aa366aea0f94fb680397fbb8a3b4f7d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/B/6aa366aea0f94fb680397fbb8a3b4f7d 2024-12-07T18:14:12,497 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/d85ceda1ad36433aa9235f987d5c3bf0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/d85ceda1ad36433aa9235f987d5c3bf0 2024-12-07T18:14:12,498 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/f67601d116df42f1a69a6e2b2e8fdddd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/C/f67601d116df42f1a69a6e2b2e8fdddd 2024-12-07T18:14:12,501 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/recovered.edits/625.seqid to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72/recovered.edits/625.seqid 2024-12-07T18:14:12,502 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/66753c009d64518d3f4dfefd580afd72 2024-12-07T18:14:12,502 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T18:14:12,506 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:12,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-07T18:14:12,515 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T18:14:12,547 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T18:14:12,548 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:12,548 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T18:14:12,549 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733595252548"}]},"ts":"9223372036854775807"} 2024-12-07T18:14:12,553 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T18:14:12,553 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 66753c009d64518d3f4dfefd580afd72, NAME => 'TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T18:14:12,553 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T18:14:12,553 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733595252553"}]},"ts":"9223372036854775807"} 2024-12-07T18:14:12,555 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T18:14:12,556 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:12,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-12-07T18:14:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-07T18:14:12,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T18:14:12,589 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:12,599 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;92acd7e2389b:33947-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-17801953_22 at /127.0.0.1:59504 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 234) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7170 (was 6979) - AvailableMemoryMB LEAK? - 2024-12-07T18:14:12,608 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=7170 2024-12-07T18:14:12,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T18:14:12,609 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:14:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:12,611 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T18:14:12,612 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:12,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-07T18:14:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-07T18:14:12,612 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T18:14:12,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741954_1130 (size=1061) 2024-12-07T18:14:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-07T18:14:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-07T18:14:13,020 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68 2024-12-07T18:14:13,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741955_1131 (size=53) 2024-12-07T18:14:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-07T18:14:13,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:14:13,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3d520264937e773a7192a548dec4f0b4, disabling compactions & flushes 2024-12-07T18:14:13,427 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:13,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:13,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. after waiting 0 ms 2024-12-07T18:14:13,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:13,427 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:13,427 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3d520264937e773a7192a548dec4f0b4: Waiting for close lock at 1733595253427Disabling compacts and flushes for region at 1733595253427Disabling writes for close at 1733595253427Writing region close event to WAL at 1733595253427Closed at 1733595253427 2024-12-07T18:14:13,428 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T18:14:13,429 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733595253428"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733595253428"}]},"ts":"1733595253428"} 2024-12-07T18:14:13,430 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T18:14:13,431 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T18:14:13,431 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595253431"}]},"ts":"1733595253431"} 2024-12-07T18:14:13,433 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T18:14:13,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, ASSIGN}] 2024-12-07T18:14:13,434 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, ASSIGN 2024-12-07T18:14:13,435 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, ASSIGN; state=OFFLINE, location=92acd7e2389b,33947,1733595217075; forceNewPlan=false, retain=false 2024-12-07T18:14:13,585 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3d520264937e773a7192a548dec4f0b4, regionState=OPENING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:13,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, ASSIGN because future has completed 2024-12-07T18:14:13,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:14:13,743 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:13,743 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:14:13,744 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,744 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:14:13,744 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,744 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,745 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,746 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:14:13,747 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d520264937e773a7192a548dec4f0b4 columnFamilyName A 2024-12-07T18:14:13,748 DEBUG [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:13,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-07T18:14:13,748 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(327): Store=3d520264937e773a7192a548dec4f0b4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:14:13,748 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,749 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:14:13,749 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d520264937e773a7192a548dec4f0b4 columnFamilyName B 2024-12-07T18:14:13,749 DEBUG [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:13,750 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(327): Store=3d520264937e773a7192a548dec4f0b4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:14:13,750 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,751 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:14:13,751 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d520264937e773a7192a548dec4f0b4 columnFamilyName C 2024-12-07T18:14:13,751 DEBUG [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:13,752 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(327): Store=3d520264937e773a7192a548dec4f0b4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:14:13,752 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:13,752 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,753 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,753 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,754 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,754 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,754 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:14:13,755 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,757 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:14:13,757 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 3d520264937e773a7192a548dec4f0b4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74247103, jitterRate=0.1063680499792099}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:14:13,758 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:13,758 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 3d520264937e773a7192a548dec4f0b4: Running coprocessor pre-open hook at 1733595253744Writing region info on filesystem at 1733595253744Initializing all the Stores at 1733595253745 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595253745Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595253745Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595253745Cleaning up temporary data from old regions at 1733595253754 (+9 ms)Running coprocessor post-open hooks at 1733595253758 (+4 ms)Region opened successfully at 1733595253758 2024-12-07T18:14:13,759 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., pid=32, masterSystemTime=1733595253740 2024-12-07T18:14:13,761 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:13,761 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:13,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=3d520264937e773a7192a548dec4f0b4, regionState=OPEN, openSeqNum=2, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:13,763 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:14:13,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-07T18:14:13,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 in 176 msec 2024-12-07T18:14:13,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-07T18:14:13,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, ASSIGN in 333 msec 2024-12-07T18:14:13,769 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T18:14:13,769 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595253769"}]},"ts":"1733595253769"} 2024-12-07T18:14:13,770 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T18:14:13,771 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T18:14:13,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-12-07T18:14:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-07T18:14:14,759 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:14,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:14,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:14,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:14,761 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7192040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:14,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:14,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:14,764 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:14,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37421322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:14,765 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:14,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:14,769 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T18:14:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 92acd7e2389b,38531,1733595216342 2024-12-07T18:14:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3df6455a 2024-12-07T18:14:14,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T18:14:14,770 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T18:14:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T18:14:14,775 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:14:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:14,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741956_1132 (size=1097) 2024-12-07T18:14:15,193 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-07T18:14:15,193 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-07T18:14:15,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T18:14:15,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, REOPEN/MOVE}] 2024-12-07T18:14:15,204 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, REOPEN/MOVE 2024-12-07T18:14:15,204 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3d520264937e773a7192a548dec4f0b4, regionState=CLOSING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:15,206 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, REOPEN/MOVE because future has completed 2024-12-07T18:14:15,207 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T18:14:15,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:14:15,359 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,359 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T18:14:15,359 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 3d520264937e773a7192a548dec4f0b4, disabling compactions & flushes 2024-12-07T18:14:15,359 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:15,359 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:15,359 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. after waiting 0 ms 2024-12-07T18:14:15,359 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:15,363 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T18:14:15,364 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:15,364 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 3d520264937e773a7192a548dec4f0b4: Waiting for close lock at 1733595255359Running coprocessor pre-close hooks at 1733595255359Disabling compacts and flushes for region at 1733595255359Disabling writes for close at 1733595255359Writing region close event to WAL at 1733595255360 (+1 ms)Running coprocessor post-close hooks at 1733595255364 (+4 ms)Closed at 1733595255364 2024-12-07T18:14:15,364 WARN [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 3d520264937e773a7192a548dec4f0b4 to self. 2024-12-07T18:14:15,366 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,366 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3d520264937e773a7192a548dec4f0b4, regionState=CLOSED 2024-12-07T18:14:15,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:14:15,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-07T18:14:15,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 in 162 msec 2024-12-07T18:14:15,371 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, REOPEN/MOVE; state=CLOSED, location=92acd7e2389b,33947,1733595217075; forceNewPlan=false, retain=true 2024-12-07T18:14:15,522 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3d520264937e773a7192a548dec4f0b4, regionState=OPENING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:15,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, REOPEN/MOVE because future has completed 2024-12-07T18:14:15,524 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:14:15,679 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:15,679 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:14:15,679 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,679 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:14:15,679 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,679 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,681 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,682 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:14:15,687 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d520264937e773a7192a548dec4f0b4 columnFamilyName A 2024-12-07T18:14:15,688 DEBUG [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:15,689 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(327): Store=3d520264937e773a7192a548dec4f0b4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:14:15,689 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,690 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:14:15,690 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d520264937e773a7192a548dec4f0b4 columnFamilyName B 2024-12-07T18:14:15,690 DEBUG [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:15,690 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(327): Store=3d520264937e773a7192a548dec4f0b4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:14:15,690 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,691 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:14:15,691 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d520264937e773a7192a548dec4f0b4 columnFamilyName C 2024-12-07T18:14:15,691 DEBUG [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:15,691 INFO [StoreOpener-3d520264937e773a7192a548dec4f0b4-1 {}] regionserver.HStore(327): Store=3d520264937e773a7192a548dec4f0b4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:14:15,691 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:15,692 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,692 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,693 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,694 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,694 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,694 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:14:15,695 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,696 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 3d520264937e773a7192a548dec4f0b4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75323311, jitterRate=0.1224047988653183}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:14:15,696 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,698 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 3d520264937e773a7192a548dec4f0b4: Running coprocessor pre-open hook at 1733595255679Writing region info on filesystem at 1733595255679Initializing all the Stores at 1733595255680 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595255680Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595255680Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595255680Cleaning up temporary data from old regions at 1733595255694 (+14 ms)Running coprocessor post-open hooks at 1733595255696 (+2 ms)Region opened successfully at 1733595255698 (+2 ms) 2024-12-07T18:14:15,699 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., pid=37, masterSystemTime=1733595255676 2024-12-07T18:14:15,701 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:15,701 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:15,701 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=3d520264937e773a7192a548dec4f0b4, regionState=OPEN, openSeqNum=5, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:15,703 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:14:15,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-07T18:14:15,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 in 180 msec 2024-12-07T18:14:15,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-07T18:14:15,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, REOPEN/MOVE in 503 msec 2024-12-07T18:14:15,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-07T18:14:15,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 514 msec 2024-12-07T18:14:15,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 936 msec 2024-12-07T18:14:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-07T18:14:15,719 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:15,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:15,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:15,721 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:15,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7c3edf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:15,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:15,723 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554d2b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:15,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:15,725 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1389b68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:15,727 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c5c7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:15,729 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:15,729 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:15,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a7bfaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:15,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:15,731 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:15,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3edd8f8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:15,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:15,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62699258, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:15,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:15,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:15,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391e49e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:15,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:15,737 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100cfb7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:15,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:15,738 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@608c73be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:15,745 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:15,746 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7dac9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:15,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:15,748 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753aad34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:15,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:15,752 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700545fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:15,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:15,754 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd0e0eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:15,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:15,756 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e90d2cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:15,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:15,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:15,757 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1701504c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:15,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:15,759 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@687cce8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:15,761 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a87beba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:15,762 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:15,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:15,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:15,766 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-07T18:14:15,768 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:15,768 DEBUG [Thread-679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:15,768 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-07T18:14:15,769 DEBUG [Thread-666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:15,769 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:14:15,770 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:15,770 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:15,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:15,772 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:15,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:15,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:15,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:15,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:14:15,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:15,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:14:15,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:15,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:15,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:14:15,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:14:15,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:15,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:14:15,785 DEBUG [Thread-668 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:15,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:14:15,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:14:15,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:14:15,796 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:15,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:15,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:15,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:15,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:15,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:15,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:15,797 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:15,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:15,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:15,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:14:15,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412078f9c821187dd4924afd1185cce06062b_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595255795/Put/seqid=0 2024-12-07T18:14:15,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733595255817, max timestamp=1733595255845Num uniques 859; 2024-12-07T18:14:15,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T18:14:15,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733595255817, max timestamp=1733595255845Num uniques 859; 2024-12-07T18:14:15,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-07T18:14:15,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595255825, max timestamp=1733595255845Num uniques 859; 2024-12-07T18:14:15,854 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:15,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741957_1133 (size=12154) 2024-12-07T18:14:15,863 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:15,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412078f9c821187dd4924afd1185cce06062b_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412078f9c821187dd4924afd1185cce06062b_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:15,870 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/3395dd52baa640e6aa23d6bec190937b, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:15,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-07T18:14:15,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/3395dd52baa640e6aa23d6bec190937b is 175, key is test_row_0/A:col10/1733595255795/Put/seqid=0 2024-12-07T18:14:15,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741958_1134 (size=30951) 2024-12-07T18:14:15,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:15,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595315891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:15,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:15,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595315891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:15,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:15,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595315894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:15,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595315894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:15,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:15,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595315903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:15,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:15,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:15,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:15,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:15,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:15,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:15,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:15,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595316009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595316009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595316010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595316010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595316018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-07T18:14:16,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595316219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595316219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595316219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595316228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595316230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,292 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/3395dd52baa640e6aa23d6bec190937b 2024-12-07T18:14:16,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/b58f8a74c1b445a8907233fe3f0f44dd is 50, key is test_row_0/B:col10/1733595255795/Put/seqid=0 2024-12-07T18:14:16,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741959_1135 (size=9657) 2024-12-07T18:14:16,371 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/b58f8a74c1b445a8907233fe3f0f44dd 2024-12-07T18:14:16,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:16,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:16,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-07T18:14:16,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/0407c2f868b744cfb5143de1d2710eab is 50, key is test_row_0/C:col10/1733595255795/Put/seqid=0 2024-12-07T18:14:16,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741960_1136 (size=9657) 2024-12-07T18:14:16,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/0407c2f868b744cfb5143de1d2710eab 2024-12-07T18:14:16,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/3395dd52baa640e6aa23d6bec190937b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/3395dd52baa640e6aa23d6bec190937b 2024-12-07T18:14:16,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/3395dd52baa640e6aa23d6bec190937b, entries=150, sequenceid=15, filesize=30.2 K 2024-12-07T18:14:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/b58f8a74c1b445a8907233fe3f0f44dd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/b58f8a74c1b445a8907233fe3f0f44dd 2024-12-07T18:14:16,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/b58f8a74c1b445a8907233fe3f0f44dd, entries=100, sequenceid=15, filesize=9.4 K 2024-12-07T18:14:16,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/0407c2f868b744cfb5143de1d2710eab as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/0407c2f868b744cfb5143de1d2710eab 2024-12-07T18:14:16,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/0407c2f868b744cfb5143de1d2710eab, entries=100, sequenceid=15, filesize=9.4 K 2024-12-07T18:14:16,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3d520264937e773a7192a548dec4f0b4 in 713ms, sequenceid=15, compaction requested=false 2024-12-07T18:14:16,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:16,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-07T18:14:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:16,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:16,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:16,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:16,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:16,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e2838460d9c8436d9833043a3b78b131_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595256534/Put/seqid=0 2024-12-07T18:14:16,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595316563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595316564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595316565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595316566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595316566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741961_1137 (size=17034) 2024-12-07T18:14:16,595 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:16,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e2838460d9c8436d9833043a3b78b131_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e2838460d9c8436d9833043a3b78b131_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:16,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/794c223d7fed486da9b17495fed93034, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:16,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/794c223d7fed486da9b17495fed93034 is 175, key is test_row_0/A:col10/1733595256534/Put/seqid=0 2024-12-07T18:14:16,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741962_1138 (size=48139) 2024-12-07T18:14:16,618 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/794c223d7fed486da9b17495fed93034 2024-12-07T18:14:16,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/9e3edf303e264978b7c3b16db54bfeb0 is 50, key is test_row_0/B:col10/1733595256534/Put/seqid=0 2024-12-07T18:14:16,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595316678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595316678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595316679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741963_1139 (size=12001) 2024-12-07T18:14:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595316679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595316688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:16,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:16,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:16,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:16,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:16,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:16,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595316889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595316889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595316889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595316898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-07T18:14:16,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:16,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595316912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:16,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/9e3edf303e264978b7c3b16db54bfeb0 2024-12-07T18:14:17,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/7516bc76d0ba45df8007fcfc94d5067f is 50, key is test_row_0/C:col10/1733595256534/Put/seqid=0 2024-12-07T18:14:17,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741964_1140 (size=12001) 2024-12-07T18:14:17,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/7516bc76d0ba45df8007fcfc94d5067f 2024-12-07T18:14:17,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/794c223d7fed486da9b17495fed93034 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/794c223d7fed486da9b17495fed93034 2024-12-07T18:14:17,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/794c223d7fed486da9b17495fed93034, entries=250, sequenceid=52, filesize=47.0 K 2024-12-07T18:14:17,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/9e3edf303e264978b7c3b16db54bfeb0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/9e3edf303e264978b7c3b16db54bfeb0 2024-12-07T18:14:17,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:17,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:17,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/9e3edf303e264978b7c3b16db54bfeb0, entries=150, sequenceid=52, filesize=11.7 K 2024-12-07T18:14:17,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/7516bc76d0ba45df8007fcfc94d5067f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/7516bc76d0ba45df8007fcfc94d5067f 2024-12-07T18:14:17,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/7516bc76d0ba45df8007fcfc94d5067f, entries=150, sequenceid=52, filesize=11.7 K 2024-12-07T18:14:17,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 3d520264937e773a7192a548dec4f0b4 in 644ms, sequenceid=52, compaction requested=false 2024-12-07T18:14:17,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:17,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:17,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T18:14:17,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:17,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:17,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120788c98bc214b14dd38414ef8bed6f753f_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595257211/Put/seqid=0 2024-12-07T18:14:17,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595317244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595317246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595317250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595317255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595317256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741965_1141 (size=12154) 2024-12-07T18:14:17,278 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:17,286 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120788c98bc214b14dd38414ef8bed6f753f_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120788c98bc214b14dd38414ef8bed6f753f_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:17,287 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/aba59a87d9f842faa7c2d509c800d0db, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:17,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/aba59a87d9f842faa7c2d509c800d0db is 175, key is test_row_0/A:col10/1733595257211/Put/seqid=0 2024-12-07T18:14:17,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:17,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:17,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741966_1142 (size=30955) 2024-12-07T18:14:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,326 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/aba59a87d9f842faa7c2d509c800d0db 2024-12-07T18:14:17,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/7d6e81a0fe144bb5816420d4a7551d2a is 50, key is test_row_0/B:col10/1733595257211/Put/seqid=0 2024-12-07T18:14:17,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595317359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595317369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595317369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595317369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595317369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741967_1143 (size=12001) 2024-12-07T18:14:17,466 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T18:14:17,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:17,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:17,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595317579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595317579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595317579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595317578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595317589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:17,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:17,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:17,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:17,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/7d6e81a0fe144bb5816420d4a7551d2a 2024-12-07T18:14:17,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/c918e01c3de64f3ea5cacbc1921efd73 is 50, key is test_row_0/C:col10/1733595257211/Put/seqid=0 2024-12-07T18:14:17,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741968_1144 (size=12001) 2024-12-07T18:14:17,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/c918e01c3de64f3ea5cacbc1921efd73 2024-12-07T18:14:17,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/aba59a87d9f842faa7c2d509c800d0db as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/aba59a87d9f842faa7c2d509c800d0db 2024-12-07T18:14:17,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/aba59a87d9f842faa7c2d509c800d0db, entries=150, sequenceid=64, filesize=30.2 K 2024-12-07T18:14:17,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/7d6e81a0fe144bb5816420d4a7551d2a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7d6e81a0fe144bb5816420d4a7551d2a 2024-12-07T18:14:17,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7d6e81a0fe144bb5816420d4a7551d2a, entries=150, sequenceid=64, filesize=11.7 K 2024-12-07T18:14:17,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/c918e01c3de64f3ea5cacbc1921efd73 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c918e01c3de64f3ea5cacbc1921efd73 2024-12-07T18:14:17,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595317888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:17,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:17,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c918e01c3de64f3ea5cacbc1921efd73, entries=150, sequenceid=64, filesize=11.7 K 2024-12-07T18:14:17,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3d520264937e773a7192a548dec4f0b4 in 693ms, sequenceid=64, compaction requested=true 2024-12-07T18:14:17,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:17,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:17,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:17,908 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:17,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:17,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:17,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:17,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:17,908 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:17,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-07T18:14:17,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595257240, max timestamp=1733595257906Num uniques 859; 2024-12-07T18:14:17,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:17,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595257240, max timestamp=1733595257906Num uniques 859; 2024-12-07T18:14:17,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:17,912 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595257240, max timestamp=1733595257906Num uniques 859; 2024-12-07T18:14:17,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:17,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:17,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:17,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:17,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:17,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:17,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:17,914 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:17,914 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/B is initiating minor compaction (all files) 2024-12-07T18:14:17,914 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/B in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,914 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/b58f8a74c1b445a8907233fe3f0f44dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/9e3edf303e264978b7c3b16db54bfeb0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7d6e81a0fe144bb5816420d4a7551d2a] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=32.9 K 2024-12-07T18:14:17,915 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:17,915 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/A is initiating minor compaction (all files) 2024-12-07T18:14:17,915 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/A in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,915 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/3395dd52baa640e6aa23d6bec190937b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/794c223d7fed486da9b17495fed93034, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/aba59a87d9f842faa7c2d509c800d0db] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=107.5 K 2024-12-07T18:14:17,915 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,915 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/3395dd52baa640e6aa23d6bec190937b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/794c223d7fed486da9b17495fed93034, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/aba59a87d9f842faa7c2d509c800d0db] 2024-12-07T18:14:17,916 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting b58f8a74c1b445a8907233fe3f0f44dd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733595255788 2024-12-07T18:14:17,917 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e3edf303e264978b7c3b16db54bfeb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733595255900 2024-12-07T18:14:17,917 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3395dd52baa640e6aa23d6bec190937b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733595255784 2024-12-07T18:14:17,917 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d6e81a0fe144bb5816420d4a7551d2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595257201 2024-12-07T18:14:17,918 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 794c223d7fed486da9b17495fed93034, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733595255886 2024-12-07T18:14:17,919 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba59a87d9f842faa7c2d509c800d0db, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595257201 2024-12-07T18:14:17,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-07T18:14:17,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:17,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:17,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:17,942 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#B#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:17,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,943 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/5535a08381be49f4adaadaefba8fb3ba is 50, key is test_row_0/B:col10/1733595257211/Put/seqid=0 2024-12-07T18:14:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:17,957 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:17,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733595257940, max timestamp=1733595257964Num uniques 859; 2024-12-07T18:14:17,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-07T18:14:17,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733595257940, max timestamp=1733595257964Num uniques 859; 2024-12-07T18:14:17,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-07T18:14:17,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595257955, max timestamp=1733595257964Num uniques 859; 2024-12-07T18:14:17,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:17,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120701c94e934eb642108d69c3f9c4f77202_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595257240/Put/seqid=0 2024-12-07T18:14:17,984 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207b4c186a5f39f4631aa6eecff4402fcb1_3d520264937e773a7192a548dec4f0b4 store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:17,989 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207b4c186a5f39f4631aa6eecff4402fcb1_3d520264937e773a7192a548dec4f0b4, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:17,989 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b4c186a5f39f4631aa6eecff4402fcb1_3d520264937e773a7192a548dec4f0b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:18,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741969_1145 (size=12104) 2024-12-07T18:14:18,020 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/5535a08381be49f4adaadaefba8fb3ba as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/5535a08381be49f4adaadaefba8fb3ba 2024-12-07T18:14:18,026 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/B of 3d520264937e773a7192a548dec4f0b4 into 5535a08381be49f4adaadaefba8fb3ba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:18,026 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:18,026 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/B, priority=13, startTime=1733595257908; duration=0sec 2024-12-07T18:14:18,027 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:18,027 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:B 2024-12-07T18:14:18,027 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:18,028 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:18,029 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/C is initiating minor compaction (all files) 2024-12-07T18:14:18,029 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/C in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:18,029 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/0407c2f868b744cfb5143de1d2710eab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/7516bc76d0ba45df8007fcfc94d5067f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c918e01c3de64f3ea5cacbc1921efd73] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=32.9 K 2024-12-07T18:14:18,031 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 0407c2f868b744cfb5143de1d2710eab, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733595255788 2024-12-07T18:14:18,032 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7516bc76d0ba45df8007fcfc94d5067f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733595255900 2024-12-07T18:14:18,032 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting c918e01c3de64f3ea5cacbc1921efd73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595257201 2024-12-07T18:14:18,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741971_1147 (size=4469) 2024-12-07T18:14:18,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595318033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595318033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595318037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733595258007, max timestamp=1733595258045Num uniques 1218; 2024-12-07T18:14:18,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=14558, suffix heap size=154640,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=318 2024-12-07T18:14:18,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733595258007, max timestamp=1733595258045Num uniques 1218; 2024-12-07T18:14:18,052 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55778, new segment data size=14558, suffix heap size=154640,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1218, new segment cells count=318 2024-12-07T18:14:18,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733595258007, max timestamp=1733595258045Num uniques 1018; 2024-12-07T18:14:18,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-12-07T18:14:18,053 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#A#compaction#127 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,054 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/168b986cc5724e5982ee5116b2736907 is 175, key is test_row_0/A:col10/1733595257211/Put/seqid=0 2024-12-07T18:14:18,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741970_1146 (size=17034) 2024-12-07T18:14:18,055 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#C#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:18,056 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/d6dd97db0d2f4da89b75c19be121b861 is 50, key is test_row_0/C:col10/1733595257211/Put/seqid=0 2024-12-07T18:14:18,057 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:18,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120701c94e934eb642108d69c3f9c4f77202_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120701c94e934eb642108d69c3f9c4f77202_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:18,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/dbeb9f9749ac4d9e91afe5c4f3374de3, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:18,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/dbeb9f9749ac4d9e91afe5c4f3374de3 is 175, key is test_row_0/A:col10/1733595257240/Put/seqid=0 2024-12-07T18:14:18,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:18,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:18,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:18,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:18,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:18,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:18,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741973_1149 (size=12104) 2024-12-07T18:14:18,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741972_1148 (size=31058) 2024-12-07T18:14:18,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741974_1150 (size=48139) 2024-12-07T18:14:18,163 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/dbeb9f9749ac4d9e91afe5c4f3374de3 2024-12-07T18:14:18,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595318170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/cb7742c0a04c4f4e9ebd7641259eb1f3 is 50, key is test_row_0/B:col10/1733595257240/Put/seqid=0 2024-12-07T18:14:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595318170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595318167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595318186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741975_1151 (size=12001) 2024-12-07T18:14:18,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/cb7742c0a04c4f4e9ebd7641259eb1f3 2024-12-07T18:14:18,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:18,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:18,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:18,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:18,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:18,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:18,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:18,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/623129636b2c4d54b649d41ff39cda02 is 50, key is test_row_0/C:col10/1733595257240/Put/seqid=0 2024-12-07T18:14:18,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741976_1152 (size=12001) 2024-12-07T18:14:18,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/623129636b2c4d54b649d41ff39cda02 2024-12-07T18:14:18,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/dbeb9f9749ac4d9e91afe5c4f3374de3 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/dbeb9f9749ac4d9e91afe5c4f3374de3 2024-12-07T18:14:18,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/dbeb9f9749ac4d9e91afe5c4f3374de3, entries=250, sequenceid=86, filesize=47.0 K 2024-12-07T18:14:18,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/cb7742c0a04c4f4e9ebd7641259eb1f3 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb7742c0a04c4f4e9ebd7641259eb1f3 2024-12-07T18:14:18,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb7742c0a04c4f4e9ebd7641259eb1f3, entries=150, sequenceid=86, filesize=11.7 K 2024-12-07T18:14:18,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/623129636b2c4d54b649d41ff39cda02 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/623129636b2c4d54b649d41ff39cda02 2024-12-07T18:14:18,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595318299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595318299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595318299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595318299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/623129636b2c4d54b649d41ff39cda02, entries=150, sequenceid=86, filesize=11.7 K 2024-12-07T18:14:18,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 3d520264937e773a7192a548dec4f0b4 in 416ms, sequenceid=86, compaction requested=false 2024-12-07T18:14:18,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:18,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-07T18:14:18,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:18,403 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=154.31 KB heapSize=429.84 KB 2024-12-07T18:14:18,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:18,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:18,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:18,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:18,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:18,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:18,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:18,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412075b8449747dbc442da15a6b47fb525e9d_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595258166/Put/seqid=0 2024-12-07T18:14:18,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741977_1153 (size=12304) 2024-12-07T18:14:18,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595318457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595318518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595318518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595318519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595318519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,534 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/d6dd97db0d2f4da89b75c19be121b861 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/d6dd97db0d2f4da89b75c19be121b861 2024-12-07T18:14:18,538 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/168b986cc5724e5982ee5116b2736907 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/168b986cc5724e5982ee5116b2736907 2024-12-07T18:14:18,547 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/C of 3d520264937e773a7192a548dec4f0b4 into d6dd97db0d2f4da89b75c19be121b861(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:18,547 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:18,547 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/C, priority=13, startTime=1733595257908; duration=0sec 2024-12-07T18:14:18,548 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:18,548 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:C 2024-12-07T18:14:18,554 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/A of 3d520264937e773a7192a548dec4f0b4 into 168b986cc5724e5982ee5116b2736907(size=30.3 K), total size for store is 77.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:18,554 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:18,554 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/A, priority=13, startTime=1733595257908; duration=0sec 2024-12-07T18:14:18,554 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:18,554 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:A 2024-12-07T18:14:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595318605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595318819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595318838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595318839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,847 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412075b8449747dbc442da15a6b47fb525e9d_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412075b8449747dbc442da15a6b47fb525e9d_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:18,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595318839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:18,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/fbe7496f1b32428d978cd11741c36cb2, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:18,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/fbe7496f1b32428d978cd11741c36cb2 is 175, key is test_row_0/A:col10/1733595258166/Put/seqid=0 2024-12-07T18:14:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595318841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:18,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:18,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741978_1154 (size=31105) 2024-12-07T18:14:19,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595319128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:19,302 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/fbe7496f1b32428d978cd11741c36cb2 2024-12-07T18:14:19,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/f8d4ee9731ac42318c5013b620c747f8 is 50, key is test_row_0/B:col10/1733595258166/Put/seqid=0 2024-12-07T18:14:19,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741979_1155 (size=12151) 2024-12-07T18:14:19,351 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/f8d4ee9731ac42318c5013b620c747f8 2024-12-07T18:14:19,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595319349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:19,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595319349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,356 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:19,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595319358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:19,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595319358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:19,363 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:19,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/df1bcf49fdb14895a6b3fa12b66d7b5e is 50, key is test_row_0/C:col10/1733595258166/Put/seqid=0 2024-12-07T18:14:19,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741980_1156 (size=12151) 2024-12-07T18:14:19,379 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/df1bcf49fdb14895a6b3fa12b66d7b5e 2024-12-07T18:14:19,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/fbe7496f1b32428d978cd11741c36cb2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/fbe7496f1b32428d978cd11741c36cb2 2024-12-07T18:14:19,393 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/fbe7496f1b32428d978cd11741c36cb2, entries=150, sequenceid=141, filesize=30.4 K 2024-12-07T18:14:19,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/f8d4ee9731ac42318c5013b620c747f8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/f8d4ee9731ac42318c5013b620c747f8 2024-12-07T18:14:19,404 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/f8d4ee9731ac42318c5013b620c747f8, entries=150, sequenceid=141, filesize=11.9 K 2024-12-07T18:14:19,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/df1bcf49fdb14895a6b3fa12b66d7b5e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/df1bcf49fdb14895a6b3fa12b66d7b5e 2024-12-07T18:14:19,414 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/df1bcf49fdb14895a6b3fa12b66d7b5e, entries=150, sequenceid=141, filesize=11.9 K 2024-12-07T18:14:19,416 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~429.80 KB/440112, currentSize=33.54 KB/34350 for 3d520264937e773a7192a548dec4f0b4 in 1013ms, sequenceid=141, compaction requested=true 2024-12-07T18:14:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:19,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-07T18:14:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-07T18:14:19,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-07T18:14:19,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6480 sec 2024-12-07T18:14:19,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.6560 sec 2024-12-07T18:14:19,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:14:19,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:19,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:19,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:19,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:19,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:19,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:19,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e73138f8c040493986226bdffe2fb62f_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595258446/Put/seqid=0 2024-12-07T18:14:19,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741981_1157 (size=14794) 2024-12-07T18:14:19,732 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:19,740 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e73138f8c040493986226bdffe2fb62f_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e73138f8c040493986226bdffe2fb62f_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:19,741 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/da2ec569a4e14e4aba09b9e8618218ac, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:19,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/da2ec569a4e14e4aba09b9e8618218ac is 175, key is test_row_0/A:col10/1733595258446/Put/seqid=0 2024-12-07T18:14:19,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741982_1158 (size=39749) 2024-12-07T18:14:19,753 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/da2ec569a4e14e4aba09b9e8618218ac 2024-12-07T18:14:19,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/a5a2bf63b63b4ab3bbcf3651b754500e is 50, key is test_row_0/B:col10/1733595258446/Put/seqid=0 2024-12-07T18:14:19,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741983_1159 (size=12151) 2024-12-07T18:14:19,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595259781, max timestamp=1733595259817Num uniques 859; 2024-12-07T18:14:19,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:14:19,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595259781, max timestamp=1733595259817Num uniques 859; 2024-12-07T18:14:19,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:14:19,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595259781, max timestamp=1733595259817Num uniques 859; 2024-12-07T18:14:19,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:14:19,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595319914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-07T18:14:19,949 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:19,949 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-07T18:14:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T18:14:19,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:19,952 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:19,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:20,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:20,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595320019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:20,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:20,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T18:14:20,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-07T18:14:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/a5a2bf63b63b4ab3bbcf3651b754500e 2024-12-07T18:14:20,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/f3bbe0cfeb0e48d88a27e6b2e6e03f20 is 50, key is test_row_0/C:col10/1733595258446/Put/seqid=0 2024-12-07T18:14:20,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741984_1160 (size=12151) 2024-12-07T18:14:20,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595320229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:20,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-07T18:14:20,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:20,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T18:14:20,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595320368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:20,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595320368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:20,370 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:20,372 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:20,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:20,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595320379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:20,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595320379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:20,392 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:20,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:20,393 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:20,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-07T18:14:20,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:20,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595320539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:20,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-07T18:14:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:20,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T18:14:20,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/f3bbe0cfeb0e48d88a27e6b2e6e03f20 2024-12-07T18:14:20,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/da2ec569a4e14e4aba09b9e8618218ac as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/da2ec569a4e14e4aba09b9e8618218ac 2024-12-07T18:14:20,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/da2ec569a4e14e4aba09b9e8618218ac, entries=200, sequenceid=153, filesize=38.8 K 2024-12-07T18:14:20,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/a5a2bf63b63b4ab3bbcf3651b754500e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/a5a2bf63b63b4ab3bbcf3651b754500e 2024-12-07T18:14:20,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/a5a2bf63b63b4ab3bbcf3651b754500e, entries=150, sequenceid=153, filesize=11.9 K 2024-12-07T18:14:20,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/f3bbe0cfeb0e48d88a27e6b2e6e03f20 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/f3bbe0cfeb0e48d88a27e6b2e6e03f20 2024-12-07T18:14:20,641 INFO [master/92acd7e2389b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-07T18:14:20,641 INFO [master/92acd7e2389b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-07T18:14:20,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/f3bbe0cfeb0e48d88a27e6b2e6e03f20, entries=150, sequenceid=153, filesize=11.9 K 2024-12-07T18:14:20,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3d520264937e773a7192a548dec4f0b4 in 1003ms, sequenceid=153, compaction requested=true 2024-12-07T18:14:20,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:20,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:20,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:20,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:20,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:20,657 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:14:20,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:20,657 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:14:20,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:20,659 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:14:20,659 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/B is initiating minor compaction (all files) 2024-12-07T18:14:20,659 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/B in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,659 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/5535a08381be49f4adaadaefba8fb3ba, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb7742c0a04c4f4e9ebd7641259eb1f3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/f8d4ee9731ac42318c5013b620c747f8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/a5a2bf63b63b4ab3bbcf3651b754500e] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=47.3 K 2024-12-07T18:14:20,660 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 5535a08381be49f4adaadaefba8fb3ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595257201 2024-12-07T18:14:20,660 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting cb7742c0a04c4f4e9ebd7641259eb1f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733595257240 2024-12-07T18:14:20,660 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150051 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:14:20,660 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/A is initiating minor compaction (all files) 2024-12-07T18:14:20,661 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/A in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,661 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting f8d4ee9731ac42318c5013b620c747f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733595258164 2024-12-07T18:14:20,661 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/168b986cc5724e5982ee5116b2736907, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/dbeb9f9749ac4d9e91afe5c4f3374de3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/fbe7496f1b32428d978cd11741c36cb2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/da2ec569a4e14e4aba09b9e8618218ac] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=146.5 K 2024-12-07T18:14:20,661 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,661 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/168b986cc5724e5982ee5116b2736907, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/dbeb9f9749ac4d9e91afe5c4f3374de3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/fbe7496f1b32428d978cd11741c36cb2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/da2ec569a4e14e4aba09b9e8618218ac] 2024-12-07T18:14:20,661 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting a5a2bf63b63b4ab3bbcf3651b754500e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733595258437 2024-12-07T18:14:20,662 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 168b986cc5724e5982ee5116b2736907, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595257201 2024-12-07T18:14:20,662 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbeb9f9749ac4d9e91afe5c4f3374de3, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733595257240 2024-12-07T18:14:20,663 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe7496f1b32428d978cd11741c36cb2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733595258164 2024-12-07T18:14:20,664 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting da2ec569a4e14e4aba09b9e8618218ac, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733595258437 2024-12-07T18:14:20,689 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:20,690 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/6387ef7662914401922350167c12f4b8 is 50, key is test_row_0/B:col10/1733595258446/Put/seqid=0 2024-12-07T18:14:20,697 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:20,704 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412074aa4311070194bb0bfdabe7fd164e891_3d520264937e773a7192a548dec4f0b4 store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:20,710 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412074aa4311070194bb0bfdabe7fd164e891_3d520264937e773a7192a548dec4f0b4, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:20,710 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412074aa4311070194bb0bfdabe7fd164e891_3d520264937e773a7192a548dec4f0b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:20,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741985_1161 (size=12391) 2024-12-07T18:14:20,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-07T18:14:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:20,723 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-07T18:14:20,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:20,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:20,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:20,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:20,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:20,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:20,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741986_1162 (size=4469) 2024-12-07T18:14:20,755 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#A#compaction#139 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:20,756 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/c50b29ba59824356bc3666d28f3a1c4f is 175, key is test_row_0/A:col10/1733595258446/Put/seqid=0 2024-12-07T18:14:20,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741987_1163 (size=31345) 2024-12-07T18:14:20,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412071449e4f447d8478e8d3ae06eb2a81ebe_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595259902/Put/seqid=0 2024-12-07T18:14:20,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741988_1164 (size=12304) 2024-12-07T18:14:20,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:20,805 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412071449e4f447d8478e8d3ae06eb2a81ebe_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071449e4f447d8478e8d3ae06eb2a81ebe_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:20,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/11d8b7e8bc27433ea63178e14ae62904, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:20,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/11d8b7e8bc27433ea63178e14ae62904 is 175, key is test_row_0/A:col10/1733595259902/Put/seqid=0 2024-12-07T18:14:20,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741989_1165 (size=31105) 2024-12-07T18:14:21,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:21,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:21,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T18:14:21,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595321095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:21,126 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/6387ef7662914401922350167c12f4b8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6387ef7662914401922350167c12f4b8 2024-12-07T18:14:21,131 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/B of 3d520264937e773a7192a548dec4f0b4 into 6387ef7662914401922350167c12f4b8(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:21,131 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:21,132 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/B, priority=12, startTime=1733595260657; duration=0sec 2024-12-07T18:14:21,132 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:21,132 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:B 2024-12-07T18:14:21,132 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:14:21,133 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:14:21,134 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/C is initiating minor compaction (all files) 2024-12-07T18:14:21,134 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/C in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:21,134 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/d6dd97db0d2f4da89b75c19be121b861, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/623129636b2c4d54b649d41ff39cda02, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/df1bcf49fdb14895a6b3fa12b66d7b5e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/f3bbe0cfeb0e48d88a27e6b2e6e03f20] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=47.3 K 2024-12-07T18:14:21,134 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting d6dd97db0d2f4da89b75c19be121b861, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595257201 2024-12-07T18:14:21,135 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 623129636b2c4d54b649d41ff39cda02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733595257240 2024-12-07T18:14:21,135 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting df1bcf49fdb14895a6b3fa12b66d7b5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733595258164 2024-12-07T18:14:21,136 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting f3bbe0cfeb0e48d88a27e6b2e6e03f20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733595258437 2024-12-07T18:14:21,146 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:21,147 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/d3b01ca4b4ef492b9ebff6fc32fbbbb7 is 50, key is test_row_0/C:col10/1733595258446/Put/seqid=0 2024-12-07T18:14:21,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741990_1166 (size=12391) 2024-12-07T18:14:21,167 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/d3b01ca4b4ef492b9ebff6fc32fbbbb7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/d3b01ca4b4ef492b9ebff6fc32fbbbb7 2024-12-07T18:14:21,172 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/C of 3d520264937e773a7192a548dec4f0b4 into d3b01ca4b4ef492b9ebff6fc32fbbbb7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:21,172 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:21,172 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/C, priority=12, startTime=1733595260657; duration=0sec 2024-12-07T18:14:21,172 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:21,172 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:C 2024-12-07T18:14:21,186 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/c50b29ba59824356bc3666d28f3a1c4f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c50b29ba59824356bc3666d28f3a1c4f 2024-12-07T18:14:21,193 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/A of 3d520264937e773a7192a548dec4f0b4 into c50b29ba59824356bc3666d28f3a1c4f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:21,193 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:21,196 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/A, priority=12, startTime=1733595260657; duration=0sec 2024-12-07T18:14:21,196 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:21,196 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:A 2024-12-07T18:14:21,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595321210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:21,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:21,231 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/11d8b7e8bc27433ea63178e14ae62904 2024-12-07T18:14:21,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/c3b7375564ef44deb3ce57dd77688a4c is 50, key is test_row_0/B:col10/1733595259902/Put/seqid=0 2024-12-07T18:14:21,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741991_1167 (size=12151) 2024-12-07T18:14:21,259 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/c3b7375564ef44deb3ce57dd77688a4c 2024-12-07T18:14:21,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/a90fab7a5ffb43b5ada4f5741a771d5d is 50, key is test_row_0/C:col10/1733595259902/Put/seqid=0 2024-12-07T18:14:21,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741992_1168 (size=12151) 2024-12-07T18:14:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595321418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:21,687 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/a90fab7a5ffb43b5ada4f5741a771d5d 2024-12-07T18:14:21,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/11d8b7e8bc27433ea63178e14ae62904 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/11d8b7e8bc27433ea63178e14ae62904 2024-12-07T18:14:21,705 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/11d8b7e8bc27433ea63178e14ae62904, entries=150, sequenceid=189, filesize=30.4 K 2024-12-07T18:14:21,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/c3b7375564ef44deb3ce57dd77688a4c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/c3b7375564ef44deb3ce57dd77688a4c 2024-12-07T18:14:21,716 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/c3b7375564ef44deb3ce57dd77688a4c, entries=150, sequenceid=189, filesize=11.9 K 2024-12-07T18:14:21,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/a90fab7a5ffb43b5ada4f5741a771d5d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/a90fab7a5ffb43b5ada4f5741a771d5d 2024-12-07T18:14:21,729 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/a90fab7a5ffb43b5ada4f5741a771d5d, entries=150, sequenceid=189, filesize=11.9 K 2024-12-07T18:14:21,730 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 3d520264937e773a7192a548dec4f0b4 in 1008ms, sequenceid=189, compaction requested=false 2024-12-07T18:14:21,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:21,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:21,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-07T18:14:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-07T18:14:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:21,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T18:14:21,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:21,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:21,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:21,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:21,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:21,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:21,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-07T18:14:21,740 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7790 sec 2024-12-07T18:14:21,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.7910 sec 2024-12-07T18:14:21,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207970f31a47b32498a8e4ab94135c76a0a_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595261732/Put/seqid=0 2024-12-07T18:14:21,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741993_1169 (size=14794) 2024-12-07T18:14:21,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595261829, max timestamp=1733595261879Num uniques 859; 2024-12-07T18:14:21,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:14:21,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595261829, max timestamp=1733595261879Num uniques 859; 2024-12-07T18:14:21,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595261829, max timestamp=1733595261879Num uniques 859; 2024-12-07T18:14:21,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:14:21,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:14:21,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595321971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:21,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:21,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:22,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595322088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-07T18:14:22,099 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:22,099 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-07T18:14:22,102 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:22,105 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T18:14:22,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:22,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:22,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207970f31a47b32498a8e4ab94135c76a0a_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207970f31a47b32498a8e4ab94135c76a0a_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:22,198 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/e8d903087ce848d8bd9e1872f95627ea, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:22,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/e8d903087ce848d8bd9e1872f95627ea is 175, key is test_row_0/A:col10/1733595261732/Put/seqid=0 2024-12-07T18:14:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T18:14:22,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741994_1170 (size=39749) 2024-12-07T18:14:22,237 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/e8d903087ce848d8bd9e1872f95627ea 2024-12-07T18:14:22,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ca7569ee992b4770bf2c684d57f231fa is 50, key is test_row_0/B:col10/1733595261732/Put/seqid=0 2024-12-07T18:14:22,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T18:14:22,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:22,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741995_1171 (size=12151) 2024-12-07T18:14:22,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ca7569ee992b4770bf2c684d57f231fa 2024-12-07T18:14:22,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595322298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:22,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/40b8f0a10add492ab3e986c47af5a92e is 50, key is test_row_0/C:col10/1733595261732/Put/seqid=0 2024-12-07T18:14:22,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741996_1172 (size=12151) 2024-12-07T18:14:22,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595322388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:22,394 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:22,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595322399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:22,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595322399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:22,404 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:22,404 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:22,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T18:14:22,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:22,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595322418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,420 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T18:14:22,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T18:14:22,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:22,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595322619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:22,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:22,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T18:14:22,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:22,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T18:14:22,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/40b8f0a10add492ab3e986c47af5a92e 2024-12-07T18:14:22,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/e8d903087ce848d8bd9e1872f95627ea as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e8d903087ce848d8bd9e1872f95627ea 2024-12-07T18:14:22,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e8d903087ce848d8bd9e1872f95627ea, entries=200, sequenceid=203, filesize=38.8 K 2024-12-07T18:14:22,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ca7569ee992b4770bf2c684d57f231fa as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ca7569ee992b4770bf2c684d57f231fa 2024-12-07T18:14:22,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ca7569ee992b4770bf2c684d57f231fa, entries=150, sequenceid=203, filesize=11.9 K 2024-12-07T18:14:22,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/40b8f0a10add492ab3e986c47af5a92e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/40b8f0a10add492ab3e986c47af5a92e 2024-12-07T18:14:22,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/40b8f0a10add492ab3e986c47af5a92e, entries=150, sequenceid=203, filesize=11.9 K 2024-12-07T18:14:22,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3d520264937e773a7192a548dec4f0b4 in 1050ms, sequenceid=203, compaction requested=true 2024-12-07T18:14:22,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:22,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:22,784 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:22,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:22,784 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:22,785 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:22,785 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/B is initiating minor compaction (all files) 2024-12-07T18:14:22,785 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/B in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,785 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6387ef7662914401922350167c12f4b8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/c3b7375564ef44deb3ce57dd77688a4c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ca7569ee992b4770bf2c684d57f231fa] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=35.8 K 2024-12-07T18:14:22,786 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102199 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:22,786 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/A is initiating minor compaction (all files) 2024-12-07T18:14:22,786 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/A in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,786 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c50b29ba59824356bc3666d28f3a1c4f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/11d8b7e8bc27433ea63178e14ae62904, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e8d903087ce848d8bd9e1872f95627ea] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=99.8 K 2024-12-07T18:14:22,786 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,786 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c50b29ba59824356bc3666d28f3a1c4f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/11d8b7e8bc27433ea63178e14ae62904, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e8d903087ce848d8bd9e1872f95627ea] 2024-12-07T18:14:22,786 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6387ef7662914401922350167c12f4b8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733595258437 2024-12-07T18:14:22,787 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c50b29ba59824356bc3666d28f3a1c4f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733595258437 2024-12-07T18:14:22,787 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting c3b7375564ef44deb3ce57dd77688a4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733595259882 2024-12-07T18:14:22,787 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11d8b7e8bc27433ea63178e14ae62904, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733595259882 2024-12-07T18:14:22,787 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ca7569ee992b4770bf2c684d57f231fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733595261084 2024-12-07T18:14:22,788 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d903087ce848d8bd9e1872f95627ea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733595261080 2024-12-07T18:14:22,807 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#B#compaction#147 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:22,811 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/3d4f5c5e4f21456abd7a5c1889035949 is 50, key is test_row_0/B:col10/1733595261732/Put/seqid=0 2024-12-07T18:14:22,817 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:22,839 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207acd85e2016634dd0ae04b02e17a8ce19_3d520264937e773a7192a548dec4f0b4 store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:22,842 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207acd85e2016634dd0ae04b02e17a8ce19_3d520264937e773a7192a548dec4f0b4, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:22,842 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207acd85e2016634dd0ae04b02e17a8ce19_3d520264937e773a7192a548dec4f0b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:22,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741997_1173 (size=12493) 2024-12-07T18:14:22,880 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/3d4f5c5e4f21456abd7a5c1889035949 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/3d4f5c5e4f21456abd7a5c1889035949 2024-12-07T18:14:22,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-07T18:14:22,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,882 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-07T18:14:22,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:22,893 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/B of 3d520264937e773a7192a548dec4f0b4 into 3d4f5c5e4f21456abd7a5c1889035949(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:22,893 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:22,893 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/B, priority=13, startTime=1733595262784; duration=0sec 2024-12-07T18:14:22,893 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:22,893 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:B 2024-12-07T18:14:22,893 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:22,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741998_1174 (size=4469) 2024-12-07T18:14:22,895 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:22,895 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/C is initiating minor compaction (all files) 2024-12-07T18:14:22,895 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/C in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:22,895 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/d3b01ca4b4ef492b9ebff6fc32fbbbb7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/a90fab7a5ffb43b5ada4f5741a771d5d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/40b8f0a10add492ab3e986c47af5a92e] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=35.8 K 2024-12-07T18:14:22,896 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b01ca4b4ef492b9ebff6fc32fbbbb7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733595258437 2024-12-07T18:14:22,896 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting a90fab7a5ffb43b5ada4f5741a771d5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1733595259882 2024-12-07T18:14:22,896 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b8f0a10add492ab3e986c47af5a92e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733595261084 2024-12-07T18:14:22,919 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:22,920 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/ed867bef6ac644a393c3be9a0c62a9a5 is 50, key is test_row_0/C:col10/1733595261732/Put/seqid=0 2024-12-07T18:14:22,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207ad674937c07844dca9824bb4eecd4067_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595261961/Put/seqid=0 2024-12-07T18:14:22,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741999_1175 (size=12493) 2024-12-07T18:14:22,971 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/ed867bef6ac644a393c3be9a0c62a9a5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/ed867bef6ac644a393c3be9a0c62a9a5 2024-12-07T18:14:22,979 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/C of 3d520264937e773a7192a548dec4f0b4 into ed867bef6ac644a393c3be9a0c62a9a5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:22,979 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:22,979 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/C, priority=13, startTime=1733595262784; duration=0sec 2024-12-07T18:14:22,979 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:22,979 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:C 2024-12-07T18:14:22,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742000_1176 (size=12304) 2024-12-07T18:14:22,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:22,989 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207ad674937c07844dca9824bb4eecd4067_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ad674937c07844dca9824bb4eecd4067_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:22,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/57a8b8103ef7477290a9d841b78845c7, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:22,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/57a8b8103ef7477290a9d841b78845c7 is 175, key is test_row_0/A:col10/1733595261961/Put/seqid=0 2024-12-07T18:14:23,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742001_1177 (size=31105) 2024-12-07T18:14:23,028 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/57a8b8103ef7477290a9d841b78845c7 2024-12-07T18:14:23,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/082e275e401c48be8eb8383fa0ddcbf6 is 50, key is test_row_0/B:col10/1733595261961/Put/seqid=0 2024-12-07T18:14:23,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742002_1178 (size=12151) 2024-12-07T18:14:23,083 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/082e275e401c48be8eb8383fa0ddcbf6 2024-12-07T18:14:23,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/b3ee8245bf684e0d8a19a73f196adb4e is 50, key is test_row_0/C:col10/1733595261961/Put/seqid=0 2024-12-07T18:14:23,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742003_1179 (size=12151) 2024-12-07T18:14:23,115 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/b3ee8245bf684e0d8a19a73f196adb4e 2024-12-07T18:14:23,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/57a8b8103ef7477290a9d841b78845c7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/57a8b8103ef7477290a9d841b78845c7 2024-12-07T18:14:23,133 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/57a8b8103ef7477290a9d841b78845c7, entries=150, sequenceid=238, filesize=30.4 K 2024-12-07T18:14:23,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:23,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:23,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/082e275e401c48be8eb8383fa0ddcbf6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/082e275e401c48be8eb8383fa0ddcbf6 2024-12-07T18:14:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,141 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/082e275e401c48be8eb8383fa0ddcbf6, entries=150, sequenceid=238, filesize=11.9 K 2024-12-07T18:14:23,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/b3ee8245bf684e0d8a19a73f196adb4e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/b3ee8245bf684e0d8a19a73f196adb4e 2024-12-07T18:14:23,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,151 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/b3ee8245bf684e0d8a19a73f196adb4e, entries=150, sequenceid=238, filesize=11.9 K 2024-12-07T18:14:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,157 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=6.71 KB/6870 for 3d520264937e773a7192a548dec4f0b4 in 275ms, sequenceid=238, compaction requested=false 2024-12-07T18:14:23,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:23,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:23,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-07T18:14:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-07T18:14:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-07T18:14:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0530 sec 2024-12-07T18:14:23,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.0650 sec 2024-12-07T18:14:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-07T18:14:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,250 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:23,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:23,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-07T18:14:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,255 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:23,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,256 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T18:14:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,296 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#A#compaction#148 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,297 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/c00f7e82c92a45c89dbef9778a2c8b64 is 175, key is test_row_0/A:col10/1733595261732/Put/seqid=0 2024-12-07T18:14:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742004_1180 (size=31447) 2024-12-07T18:14:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,351 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/c00f7e82c92a45c89dbef9778a2c8b64 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c00f7e82c92a45c89dbef9778a2c8b64 2024-12-07T18:14:23,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,358 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/A of 3d520264937e773a7192a548dec4f0b4 into c00f7e82c92a45c89dbef9778a2c8b64(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:23,358 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:23,358 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/A, priority=13, startTime=1733595262783; duration=0sec 2024-12-07T18:14:23,358 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:23,358 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:A 2024-12-07T18:14:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T18:14:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:23,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:14:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:23,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:23,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:23,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:23,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:23,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076754b815e1fa429facb06329bb77a2cb_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595263350/Put/seqid=0 2024-12-07T18:14:23,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T18:14:23,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:23,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:23,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:23,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742005_1181 (size=22268) 2024-12-07T18:14:23,527 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595263493, max timestamp=1733595263523Num uniques 859; 2024-12-07T18:14:23,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:23,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595263493, max timestamp=1733595263523Num uniques 859; 2024-12-07T18:14:23,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:23,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595263493, max timestamp=1733595263523Num uniques 859; 2024-12-07T18:14:23,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:23,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T18:14:23,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:23,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:23,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:23,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T18:14:23,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595323655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:23,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:23,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T18:14:23,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:23,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:23,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:23,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595323768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:23,860 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:23,865 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076754b815e1fa429facb06329bb77a2cb_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076754b815e1fa429facb06329bb77a2cb_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:23,866 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bf392cf45adf4eaf9eeb26011384ac40, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:23,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bf392cf45adf4eaf9eeb26011384ac40 is 175, key is test_row_0/A:col10/1733595263350/Put/seqid=0 2024-12-07T18:14:23,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T18:14:23,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:23,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:23,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:23,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:23,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742006_1182 (size=65673) 2024-12-07T18:14:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T18:14:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595323978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:24,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T18:14:24,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:24,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:24,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:24,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T18:14:24,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:24,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:24,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:24,288 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bf392cf45adf4eaf9eeb26011384ac40 2024-12-07T18:14:24,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/26e0d524db044d29b3cde0fb62f03ccb is 50, key is test_row_0/B:col10/1733595263350/Put/seqid=0 2024-12-07T18:14:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595324288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:24,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:24,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742007_1183 (size=12151) 2024-12-07T18:14:24,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/26e0d524db044d29b3cde0fb62f03ccb 2024-12-07T18:14:24,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/43a0261a2fd043858cb30bf8e5ba7156 is 50, key is test_row_0/C:col10/1733595263350/Put/seqid=0 2024-12-07T18:14:24,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742008_1184 (size=12151) 2024-12-07T18:14:24,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/43a0261a2fd043858cb30bf8e5ba7156 2024-12-07T18:14:24,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bf392cf45adf4eaf9eeb26011384ac40 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf392cf45adf4eaf9eeb26011384ac40 2024-12-07T18:14:24,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf392cf45adf4eaf9eeb26011384ac40, entries=350, sequenceid=251, filesize=64.1 K 2024-12-07T18:14:24,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/26e0d524db044d29b3cde0fb62f03ccb as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/26e0d524db044d29b3cde0fb62f03ccb 2024-12-07T18:14:24,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T18:14:24,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/26e0d524db044d29b3cde0fb62f03ccb, entries=150, sequenceid=251, filesize=11.9 K 2024-12-07T18:14:24,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:24,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:24,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:24,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/43a0261a2fd043858cb30bf8e5ba7156 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/43a0261a2fd043858cb30bf8e5ba7156 2024-12-07T18:14:24,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/43a0261a2fd043858cb30bf8e5ba7156, entries=150, sequenceid=251, filesize=11.9 K 2024-12-07T18:14:24,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3d520264937e773a7192a548dec4f0b4 in 972ms, sequenceid=251, compaction requested=true 2024-12-07T18:14:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:24,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:24,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:24,350 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:24,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:24,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:24,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:24,350 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:24,351 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:24,351 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/A is initiating minor compaction (all files) 2024-12-07T18:14:24,351 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/A in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,351 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c00f7e82c92a45c89dbef9778a2c8b64, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/57a8b8103ef7477290a9d841b78845c7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf392cf45adf4eaf9eeb26011384ac40] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=125.2 K 2024-12-07T18:14:24,351 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,351 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c00f7e82c92a45c89dbef9778a2c8b64, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/57a8b8103ef7477290a9d841b78845c7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf392cf45adf4eaf9eeb26011384ac40] 2024-12-07T18:14:24,352 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:24,352 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c00f7e82c92a45c89dbef9778a2c8b64, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733595261084 2024-12-07T18:14:24,352 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/B is initiating minor compaction (all files) 2024-12-07T18:14:24,352 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/B in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,352 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/3d4f5c5e4f21456abd7a5c1889035949, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/082e275e401c48be8eb8383fa0ddcbf6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/26e0d524db044d29b3cde0fb62f03ccb] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=35.9 K 2024-12-07T18:14:24,352 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57a8b8103ef7477290a9d841b78845c7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733595261941 2024-12-07T18:14:24,352 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d4f5c5e4f21456abd7a5c1889035949, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733595261084 2024-12-07T18:14:24,353 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf392cf45adf4eaf9eeb26011384ac40, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733595263129 2024-12-07T18:14:24,353 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 082e275e401c48be8eb8383fa0ddcbf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733595261941 2024-12-07T18:14:24,353 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e0d524db044d29b3cde0fb62f03ccb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733595263350 2024-12-07T18:14:24,365 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:24,366 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:24,367 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/48eb54076ab34ea58f906823803cdc07 is 50, key is test_row_0/B:col10/1733595263350/Put/seqid=0 2024-12-07T18:14:24,369 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412076ca79205adc244d683fe844cc25f1ab7_3d520264937e773a7192a548dec4f0b4 store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:24,372 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412076ca79205adc244d683fe844cc25f1ab7_3d520264937e773a7192a548dec4f0b4, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:24,372 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076ca79205adc244d683fe844cc25f1ab7_3d520264937e773a7192a548dec4f0b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:24,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742009_1185 (size=12595) 2024-12-07T18:14:24,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742010_1186 (size=4469) 2024-12-07T18:14:24,390 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/48eb54076ab34ea58f906823803cdc07 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/48eb54076ab34ea58f906823803cdc07 2024-12-07T18:14:24,396 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/B of 3d520264937e773a7192a548dec4f0b4 into 48eb54076ab34ea58f906823803cdc07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:24,396 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:24,396 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/B, priority=13, startTime=1733595264349; duration=0sec 2024-12-07T18:14:24,396 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:24,396 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:B 2024-12-07T18:14:24,396 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:24,398 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:24,398 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/C is initiating minor compaction (all files) 2024-12-07T18:14:24,398 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/C in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,398 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/ed867bef6ac644a393c3be9a0c62a9a5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/b3ee8245bf684e0d8a19a73f196adb4e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/43a0261a2fd043858cb30bf8e5ba7156] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=35.9 K 2024-12-07T18:14:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T18:14:24,398 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ed867bef6ac644a393c3be9a0c62a9a5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1733595261084 2024-12-07T18:14:24,399 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting b3ee8245bf684e0d8a19a73f196adb4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733595261941 2024-12-07T18:14:24,399 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 43a0261a2fd043858cb30bf8e5ba7156, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733595263350 2024-12-07T18:14:24,410 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:24,411 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/980d7128d15f45eaa6e7af61abfce758 is 50, key is test_row_0/C:col10/1733595263350/Put/seqid=0 2024-12-07T18:14:24,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742011_1187 (size=12595) 2024-12-07T18:14:24,424 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/980d7128d15f45eaa6e7af61abfce758 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/980d7128d15f45eaa6e7af61abfce758 2024-12-07T18:14:24,433 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/C of 3d520264937e773a7192a548dec4f0b4 into 980d7128d15f45eaa6e7af61abfce758(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:24,433 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:24,433 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/C, priority=13, startTime=1733595264350; duration=0sec 2024-12-07T18:14:24,433 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:24,433 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:C 2024-12-07T18:14:24,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-07T18:14:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:24,490 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-07T18:14:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:24,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073345adcc9701429cbf146b82d2d325a4_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595263649/Put/seqid=0 2024-12-07T18:14:24,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742012_1188 (size=12454) 2024-12-07T18:14:24,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:24,537 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073345adcc9701429cbf146b82d2d325a4_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073345adcc9701429cbf146b82d2d325a4_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:24,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bf676abbed7b40d583f29da54e0c5641, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:24,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bf676abbed7b40d583f29da54e0c5641 is 175, key is test_row_0/A:col10/1733595263649/Put/seqid=0 2024-12-07T18:14:24,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742013_1189 (size=31255) 2024-12-07T18:14:24,787 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#A#compaction#157 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:24,788 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/7136274d3a9741de956de0f110a5d7cd is 175, key is test_row_0/A:col10/1733595263350/Put/seqid=0 2024-12-07T18:14:24,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742014_1190 (size=31549) 2024-12-07T18:14:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:24,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:24,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595324847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:24,951 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bf676abbed7b40d583f29da54e0c5641 2024-12-07T18:14:24,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/161911049dac49ccae31f1c43b806c21 is 50, key is test_row_0/B:col10/1733595263649/Put/seqid=0 2024-12-07T18:14:24,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595324959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:24,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742015_1191 (size=12301) 2024-12-07T18:14:24,978 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/161911049dac49ccae31f1c43b806c21 2024-12-07T18:14:24,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/820772157654414fb0fe8bea80e71f3c is 50, key is test_row_0/C:col10/1733595263649/Put/seqid=0 2024-12-07T18:14:25,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742016_1192 (size=12301) 2024-12-07T18:14:25,020 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/820772157654414fb0fe8bea80e71f3c 2024-12-07T18:14:25,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bf676abbed7b40d583f29da54e0c5641 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf676abbed7b40d583f29da54e0c5641 2024-12-07T18:14:25,041 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf676abbed7b40d583f29da54e0c5641, entries=150, sequenceid=290, filesize=30.5 K 2024-12-07T18:14:25,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/161911049dac49ccae31f1c43b806c21 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/161911049dac49ccae31f1c43b806c21 2024-12-07T18:14:25,049 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/161911049dac49ccae31f1c43b806c21, entries=150, sequenceid=290, filesize=12.0 K 2024-12-07T18:14:25,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/820772157654414fb0fe8bea80e71f3c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/820772157654414fb0fe8bea80e71f3c 2024-12-07T18:14:25,060 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/820772157654414fb0fe8bea80e71f3c, entries=150, sequenceid=290, filesize=12.0 K 2024-12-07T18:14:25,062 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 3d520264937e773a7192a548dec4f0b4 in 571ms, sequenceid=290, compaction requested=false 2024-12-07T18:14:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:25,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-07T18:14:25,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-07T18:14:25,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-07T18:14:25,068 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8070 sec 2024-12-07T18:14:25,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8180 sec 2024-12-07T18:14:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:25,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T18:14:25,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:25,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:25,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:25,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:25,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:25,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:25,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207cecc63062f2b439eb9e6beb67dbbdd36_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595264831/Put/seqid=0 2024-12-07T18:14:25,199 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/7136274d3a9741de956de0f110a5d7cd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/7136274d3a9741de956de0f110a5d7cd 2024-12-07T18:14:25,205 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/A of 3d520264937e773a7192a548dec4f0b4 into 7136274d3a9741de956de0f110a5d7cd(size=30.8 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:25,206 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:25,206 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/A, priority=13, startTime=1733595264349; duration=0sec 2024-12-07T18:14:25,206 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:25,206 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:A 2024-12-07T18:14:25,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742017_1193 (size=14994) 2024-12-07T18:14:25,211 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:25,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207cecc63062f2b439eb9e6beb67dbbdd36_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207cecc63062f2b439eb9e6beb67dbbdd36_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:25,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/c2eca407342948628bf7f2080a0258d6, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:25,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/c2eca407342948628bf7f2080a0258d6 is 175, key is test_row_0/A:col10/1733595264831/Put/seqid=0 2024-12-07T18:14:25,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742018_1194 (size=39949) 2024-12-07T18:14:25,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595265254, max timestamp=1733595265290Num uniques 859; 2024-12-07T18:14:25,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:14:25,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595265254, max timestamp=1733595265290Num uniques 859; 2024-12-07T18:14:25,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:14:25,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595265254, max timestamp=1733595265290Num uniques 859; 2024-12-07T18:14:25,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:14:25,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595325378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-07T18:14:25,408 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:25,409 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:25,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-07T18:14:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T18:14:25,411 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:25,412 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:25,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:25,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595325488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T18:14:25,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-07T18:14:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:25,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:25,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:25,650 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/c2eca407342948628bf7f2080a0258d6 2024-12-07T18:14:25,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ad3647d05e2f4e25b69b85dde8bc97e1 is 50, key is test_row_0/B:col10/1733595264831/Put/seqid=0 2024-12-07T18:14:25,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742019_1195 (size=12301) 2024-12-07T18:14:25,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595325698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:25,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-07T18:14:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:25,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:25,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T18:14:25,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-07T18:14:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:25,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:26,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595326009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-07T18:14:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:26,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T18:14:26,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ad3647d05e2f4e25b69b85dde8bc97e1 2024-12-07T18:14:26,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/100fbd24e44448b8afd1216eaf2dc1c6 is 50, key is test_row_0/C:col10/1733595264831/Put/seqid=0 2024-12-07T18:14:26,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742020_1196 (size=12301) 2024-12-07T18:14:26,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/100fbd24e44448b8afd1216eaf2dc1c6 2024-12-07T18:14:26,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/c2eca407342948628bf7f2080a0258d6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c2eca407342948628bf7f2080a0258d6 2024-12-07T18:14:26,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c2eca407342948628bf7f2080a0258d6, entries=200, sequenceid=301, filesize=39.0 K 2024-12-07T18:14:26,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ad3647d05e2f4e25b69b85dde8bc97e1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ad3647d05e2f4e25b69b85dde8bc97e1 2024-12-07T18:14:26,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ad3647d05e2f4e25b69b85dde8bc97e1, entries=150, sequenceid=301, filesize=12.0 K 2024-12-07T18:14:26,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/100fbd24e44448b8afd1216eaf2dc1c6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/100fbd24e44448b8afd1216eaf2dc1c6 2024-12-07T18:14:26,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/100fbd24e44448b8afd1216eaf2dc1c6, entries=150, sequenceid=301, filesize=12.0 K 2024-12-07T18:14:26,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3d520264937e773a7192a548dec4f0b4 in 962ms, sequenceid=301, compaction requested=true 2024-12-07T18:14:26,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:26,135 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:26,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:26,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:26,135 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:26,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:26,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:26,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:26,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:26,136 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:26,136 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:26,136 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/B is initiating minor compaction (all files) 2024-12-07T18:14:26,136 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/A is initiating minor compaction (all files) 2024-12-07T18:14:26,136 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/B in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:26,136 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/A in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:26,136 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/48eb54076ab34ea58f906823803cdc07, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/161911049dac49ccae31f1c43b806c21, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ad3647d05e2f4e25b69b85dde8bc97e1] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=36.3 K 2024-12-07T18:14:26,136 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/7136274d3a9741de956de0f110a5d7cd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf676abbed7b40d583f29da54e0c5641, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c2eca407342948628bf7f2080a0258d6] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=100.3 K 2024-12-07T18:14:26,137 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:26,137 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/7136274d3a9741de956de0f110a5d7cd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf676abbed7b40d583f29da54e0c5641, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c2eca407342948628bf7f2080a0258d6] 2024-12-07T18:14:26,137 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 48eb54076ab34ea58f906823803cdc07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733595263350 2024-12-07T18:14:26,138 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7136274d3a9741de956de0f110a5d7cd, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733595263350 2024-12-07T18:14:26,138 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 161911049dac49ccae31f1c43b806c21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733595263611 2024-12-07T18:14:26,138 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf676abbed7b40d583f29da54e0c5641, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733595263611 2024-12-07T18:14:26,139 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ad3647d05e2f4e25b69b85dde8bc97e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733595264826 2024-12-07T18:14:26,139 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2eca407342948628bf7f2080a0258d6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733595264826 2024-12-07T18:14:26,157 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:26,159 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:26,159 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/dab6e8439f3744c69977104d62d49382 is 50, key is test_row_0/B:col10/1733595264831/Put/seqid=0 2024-12-07T18:14:26,161 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412075052fb325531442eb2feee7b7a280cf2_3d520264937e773a7192a548dec4f0b4 store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:26,164 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412075052fb325531442eb2feee7b7a280cf2_3d520264937e773a7192a548dec4f0b4, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:26,164 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412075052fb325531442eb2feee7b7a280cf2_3d520264937e773a7192a548dec4f0b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:26,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-07T18:14:26,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:26,178 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-07T18:14:26,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:26,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:26,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:26,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:26,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:26,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:26,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742021_1197 (size=12847) 2024-12-07T18:14:26,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742022_1198 (size=4469) 2024-12-07T18:14:26,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b34ee210695a4d55882ff579f0221728_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595265376/Put/seqid=0 2024-12-07T18:14:26,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742023_1199 (size=12454) 2024-12-07T18:14:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:26,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595326456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595326457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733595326458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,464 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8294 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595326465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,476 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595326519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T18:14:26,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595326568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595326569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595326578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,590 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:26,591 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/20f52886415d4b5da708a3f625d7be52 is 175, key is test_row_0/A:col10/1733595264831/Put/seqid=0 2024-12-07T18:14:26,594 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/dab6e8439f3744c69977104d62d49382 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/dab6e8439f3744c69977104d62d49382 2024-12-07T18:14:26,600 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/B of 3d520264937e773a7192a548dec4f0b4 into dab6e8439f3744c69977104d62d49382(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:26,600 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:26,600 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/B, priority=13, startTime=1733595266135; duration=0sec 2024-12-07T18:14:26,600 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:26,600 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:B 2024-12-07T18:14:26,601 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:26,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:26,602 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:26,602 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/C is initiating minor compaction (all files) 2024-12-07T18:14:26,603 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/C in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:26,603 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/980d7128d15f45eaa6e7af61abfce758, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/820772157654414fb0fe8bea80e71f3c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/100fbd24e44448b8afd1216eaf2dc1c6] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=36.3 K 2024-12-07T18:14:26,604 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 980d7128d15f45eaa6e7af61abfce758, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733595263350 2024-12-07T18:14:26,604 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 820772157654414fb0fe8bea80e71f3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733595263611 2024-12-07T18:14:26,605 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 100fbd24e44448b8afd1216eaf2dc1c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733595264826 2024-12-07T18:14:26,605 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b34ee210695a4d55882ff579f0221728_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b34ee210695a4d55882ff579f0221728_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:26,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/adf12d7ade1140e3bc4f8120d2e1c9c8, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:26,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/adf12d7ade1140e3bc4f8120d2e1c9c8 is 175, key is test_row_0/A:col10/1733595265376/Put/seqid=0 2024-12-07T18:14:26,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742024_1200 (size=31801) 2024-12-07T18:14:26,623 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/20f52886415d4b5da708a3f625d7be52 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/20f52886415d4b5da708a3f625d7be52 2024-12-07T18:14:26,630 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/A of 3d520264937e773a7192a548dec4f0b4 into 20f52886415d4b5da708a3f625d7be52(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:26,630 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:26,630 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/A, priority=13, startTime=1733595266135; duration=0sec 2024-12-07T18:14:26,630 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:26,630 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:A 2024-12-07T18:14:26,635 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:26,636 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/3d4b5fc6c74947788af579f9d062d7ab is 50, key is test_row_0/C:col10/1733595264831/Put/seqid=0 2024-12-07T18:14:26,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742025_1201 (size=31255) 2024-12-07T18:14:26,654 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/adf12d7ade1140e3bc4f8120d2e1c9c8 2024-12-07T18:14:26,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742026_1202 (size=12847) 2024-12-07T18:14:26,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ebfd4572a9674fcaa26f2f777ffe9c7d is 50, key is test_row_0/B:col10/1733595265376/Put/seqid=0 2024-12-07T18:14:26,676 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/3d4b5fc6c74947788af579f9d062d7ab as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3d4b5fc6c74947788af579f9d062d7ab 2024-12-07T18:14:26,683 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/C of 3d520264937e773a7192a548dec4f0b4 into 3d4b5fc6c74947788af579f9d062d7ab(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:26,683 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:26,684 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/C, priority=13, startTime=1733595266135; duration=0sec 2024-12-07T18:14:26,684 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:26,684 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:C 2024-12-07T18:14:26,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742027_1203 (size=12301) 2024-12-07T18:14:26,699 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ebfd4572a9674fcaa26f2f777ffe9c7d 2024-12-07T18:14:26,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/831e5a0cdd7a4461b65f69ff6a3e818f is 50, key is test_row_0/C:col10/1733595265376/Put/seqid=0 2024-12-07T18:14:26,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742028_1204 (size=12301) 2024-12-07T18:14:26,763 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/831e5a0cdd7a4461b65f69ff6a3e818f 2024-12-07T18:14:26,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/adf12d7ade1140e3bc4f8120d2e1c9c8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/adf12d7ade1140e3bc4f8120d2e1c9c8 2024-12-07T18:14:26,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595326779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595326779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,787 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/adf12d7ade1140e3bc4f8120d2e1c9c8, entries=150, sequenceid=336, filesize=30.5 K 2024-12-07T18:14:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ebfd4572a9674fcaa26f2f777ffe9c7d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ebfd4572a9674fcaa26f2f777ffe9c7d 2024-12-07T18:14:26,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:26,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595326788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:26,798 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ebfd4572a9674fcaa26f2f777ffe9c7d, entries=150, sequenceid=336, filesize=12.0 K 2024-12-07T18:14:26,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/831e5a0cdd7a4461b65f69ff6a3e818f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/831e5a0cdd7a4461b65f69ff6a3e818f 2024-12-07T18:14:26,808 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/831e5a0cdd7a4461b65f69ff6a3e818f, entries=150, sequenceid=336, filesize=12.0 K 2024-12-07T18:14:26,809 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 3d520264937e773a7192a548dec4f0b4 in 631ms, sequenceid=336, compaction requested=false 2024-12-07T18:14:26,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:26,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:26,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-07T18:14:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-07T18:14:26,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-07T18:14:26,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3990 sec 2024-12-07T18:14:26,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4040 sec 2024-12-07T18:14:27,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:27,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T18:14:27,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:27,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:27,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:27,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:27,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:27,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:27,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207abc4dc41877347af9386c76bedf2c8de_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595266441/Put/seqid=0 2024-12-07T18:14:27,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742029_1205 (size=14994) 2024-12-07T18:14:27,121 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:27,127 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207abc4dc41877347af9386c76bedf2c8de_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207abc4dc41877347af9386c76bedf2c8de_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:27,129 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:27,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595327125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0 is 175, key is test_row_0/A:col10/1733595266441/Put/seqid=0 2024-12-07T18:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595327128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595327130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742030_1206 (size=39949) 2024-12-07T18:14:27,142 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0 2024-12-07T18:14:27,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/e0608918e0b244b5a9e3464eecaefbd6 is 50, key is test_row_0/B:col10/1733595266441/Put/seqid=0 2024-12-07T18:14:27,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742031_1207 (size=12301) 2024-12-07T18:14:27,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595327239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595327248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595327253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595327459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595327459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595327459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595327528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,534 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-07T18:14:27,559 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:27,559 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-07T18:14:27,561 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T18:14:27,562 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:27,562 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:27,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/e0608918e0b244b5a9e3464eecaefbd6 2024-12-07T18:14:27,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/583b6073077648d6976bed26d345aa89 is 50, key is test_row_0/C:col10/1733595266441/Put/seqid=0 2024-12-07T18:14:27,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742032_1208 (size=12301) 2024-12-07T18:14:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T18:14:27,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-07T18:14:27,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:27,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:27,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:27,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:27,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:27,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595327768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,775 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595327768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:27,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595327768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:27,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:27,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-07T18:14:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:27,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T18:14:28,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/583b6073077648d6976bed26d345aa89 2024-12-07T18:14:28,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0 2024-12-07T18:14:28,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0, entries=200, sequenceid=351, filesize=39.0 K 2024-12-07T18:14:28,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/e0608918e0b244b5a9e3464eecaefbd6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/e0608918e0b244b5a9e3464eecaefbd6 2024-12-07T18:14:28,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-07T18:14:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:28,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:28,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:28,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/e0608918e0b244b5a9e3464eecaefbd6, entries=150, sequenceid=351, filesize=12.0 K 2024-12-07T18:14:28,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/583b6073077648d6976bed26d345aa89 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/583b6073077648d6976bed26d345aa89 2024-12-07T18:14:28,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/583b6073077648d6976bed26d345aa89, entries=150, sequenceid=351, filesize=12.0 K 2024-12-07T18:14:28,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3d520264937e773a7192a548dec4f0b4 in 944ms, sequenceid=351, compaction requested=true 2024-12-07T18:14:28,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:28,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:28,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:28,035 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:28,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:28,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:28,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:28,035 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:28,035 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:28,036 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:28,036 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/A is initiating minor compaction (all files) 2024-12-07T18:14:28,036 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/A in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:28,036 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/20f52886415d4b5da708a3f625d7be52, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/adf12d7ade1140e3bc4f8120d2e1c9c8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=100.6 K 2024-12-07T18:14:28,036 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:28,036 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/20f52886415d4b5da708a3f625d7be52, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/adf12d7ade1140e3bc4f8120d2e1c9c8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0] 2024-12-07T18:14:28,037 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:28,037 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/B is initiating minor compaction (all files) 2024-12-07T18:14:28,037 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/B in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:28,037 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20f52886415d4b5da708a3f625d7be52, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733595264826 2024-12-07T18:14:28,037 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/dab6e8439f3744c69977104d62d49382, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ebfd4572a9674fcaa26f2f777ffe9c7d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/e0608918e0b244b5a9e3464eecaefbd6] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=36.6 K 2024-12-07T18:14:28,037 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting adf12d7ade1140e3bc4f8120d2e1c9c8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733595265310 2024-12-07T18:14:28,038 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting dab6e8439f3744c69977104d62d49382, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733595264826 2024-12-07T18:14:28,038 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting a670c34ce71a4bf1a8a1dc2ccc3aacb0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733595266441 2024-12-07T18:14:28,038 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ebfd4572a9674fcaa26f2f777ffe9c7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733595265310 2024-12-07T18:14:28,039 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e0608918e0b244b5a9e3464eecaefbd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733595266441 2024-12-07T18:14:28,045 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:28,047 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#B#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:28,048 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/6a706aba738d4c25b55003b9647de391 is 50, key is test_row_0/B:col10/1733595266441/Put/seqid=0 2024-12-07T18:14:28,050 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207bf5fe6976e24481f874c81dcecf3f4c7_3d520264937e773a7192a548dec4f0b4 store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:28,051 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207bf5fe6976e24481f874c81dcecf3f4c7_3d520264937e773a7192a548dec4f0b4, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:28,051 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207bf5fe6976e24481f874c81dcecf3f4c7_3d520264937e773a7192a548dec4f0b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:28,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742033_1209 (size=12949) 2024-12-07T18:14:28,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742034_1210 (size=4469) 2024-12-07T18:14:28,059 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/6a706aba738d4c25b55003b9647de391 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6a706aba738d4c25b55003b9647de391 2024-12-07T18:14:28,061 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#A#compaction#174 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:28,062 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/9a19131fecc1447598a1e8da24146367 is 175, key is test_row_0/A:col10/1733595266441/Put/seqid=0 2024-12-07T18:14:28,065 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/B of 3d520264937e773a7192a548dec4f0b4 into 6a706aba738d4c25b55003b9647de391(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:28,065 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:28,065 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/B, priority=13, startTime=1733595268035; duration=0sec 2024-12-07T18:14:28,066 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:28,066 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:B 2024-12-07T18:14:28,066 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:28,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742035_1211 (size=31903) 2024-12-07T18:14:28,068 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:28,068 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/C is initiating minor compaction (all files) 2024-12-07T18:14:28,068 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/C in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:28,068 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3d4b5fc6c74947788af579f9d062d7ab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/831e5a0cdd7a4461b65f69ff6a3e818f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/583b6073077648d6976bed26d345aa89] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=36.6 K 2024-12-07T18:14:28,068 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d4b5fc6c74947788af579f9d062d7ab, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733595264826 2024-12-07T18:14:28,069 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 831e5a0cdd7a4461b65f69ff6a3e818f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1733595265310 2024-12-07T18:14:28,069 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 583b6073077648d6976bed26d345aa89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733595266441 2024-12-07T18:14:28,077 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#C#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:28,078 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/42eae6755dd8454e91a385814c63c498 is 50, key is test_row_0/C:col10/1733595266441/Put/seqid=0 2024-12-07T18:14:28,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742036_1212 (size=12949) 2024-12-07T18:14:28,087 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/42eae6755dd8454e91a385814c63c498 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/42eae6755dd8454e91a385814c63c498 2024-12-07T18:14:28,091 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/C of 3d520264937e773a7192a548dec4f0b4 into 42eae6755dd8454e91a385814c63c498(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:28,091 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:28,091 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/C, priority=13, startTime=1733595268035; duration=0sec 2024-12-07T18:14:28,091 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:28,091 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:C 2024-12-07T18:14:28,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-07T18:14:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:28,178 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T18:14:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:28,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412070cd3ac3455274fa1a6fc7ddc3cbbda7f_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595267119/Put/seqid=0 2024-12-07T18:14:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T18:14:28,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742037_1213 (size=12454) 2024-12-07T18:14:28,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:28,201 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412070cd3ac3455274fa1a6fc7ddc3cbbda7f_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070cd3ac3455274fa1a6fc7ddc3cbbda7f_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:28,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bc0b50f2c28649e197458ce85d2e52a5, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:28,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bc0b50f2c28649e197458ce85d2e52a5 is 175, key is test_row_0/A:col10/1733595267119/Put/seqid=0 2024-12-07T18:14:28,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742038_1214 (size=31255) 2024-12-07T18:14:28,212 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bc0b50f2c28649e197458ce85d2e52a5 2024-12-07T18:14:28,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ff27580b25154594bad26b91107db56a is 50, key is test_row_0/B:col10/1733595267119/Put/seqid=0 2024-12-07T18:14:28,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742039_1215 (size=12301) 2024-12-07T18:14:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:28,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:28,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595328301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595328303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595328304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595328408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,414 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595328418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595328419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,421 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,476 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/9a19131fecc1447598a1e8da24146367 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/9a19131fecc1447598a1e8da24146367 2024-12-07T18:14:28,485 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/A of 3d520264937e773a7192a548dec4f0b4 into 9a19131fecc1447598a1e8da24146367(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:28,485 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:28,485 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/A, priority=13, startTime=1733595268035; duration=0sec 2024-12-07T18:14:28,485 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:28,485 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:A 2024-12-07T18:14:28,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595328618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595328629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595328629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,651 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ff27580b25154594bad26b91107db56a 2024-12-07T18:14:28,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/c40df5e8ce9a41e19c1540be3abb9f0d is 50, key is test_row_0/C:col10/1733595267119/Put/seqid=0 2024-12-07T18:14:28,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742040_1216 (size=12301) 2024-12-07T18:14:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T18:14:28,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595328929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595328938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595328938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,099 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/c40df5e8ce9a41e19c1540be3abb9f0d 2024-12-07T18:14:29,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/bc0b50f2c28649e197458ce85d2e52a5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bc0b50f2c28649e197458ce85d2e52a5 2024-12-07T18:14:29,113 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bc0b50f2c28649e197458ce85d2e52a5, entries=150, sequenceid=373, filesize=30.5 K 2024-12-07T18:14:29,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/ff27580b25154594bad26b91107db56a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ff27580b25154594bad26b91107db56a 2024-12-07T18:14:29,118 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ff27580b25154594bad26b91107db56a, entries=150, sequenceid=373, filesize=12.0 K 2024-12-07T18:14:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/c40df5e8ce9a41e19c1540be3abb9f0d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c40df5e8ce9a41e19c1540be3abb9f0d 2024-12-07T18:14:29,135 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c40df5e8ce9a41e19c1540be3abb9f0d, entries=150, sequenceid=373, filesize=12.0 K 2024-12-07T18:14:29,136 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3d520264937e773a7192a548dec4f0b4 in 958ms, sequenceid=373, compaction requested=false 2024-12-07T18:14:29,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:29,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:29,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-07T18:14:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-07T18:14:29,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-07T18:14:29,145 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5760 sec 2024-12-07T18:14:29,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5870 sec 2024-12-07T18:14:29,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:29,440 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T18:14:29,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:29,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:29,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:29,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:29,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:29,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:29,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412072f767903a9e94aacab9246bf3c246954_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595268292/Put/seqid=0 2024-12-07T18:14:29,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742041_1217 (size=14994) 2024-12-07T18:14:29,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595329521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:29,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595329522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595329523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595329558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:29,563 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595329628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595329639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:29,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595329639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:29,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-07T18:14:29,708 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:29,709 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-07T18:14:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T18:14:29,711 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:29,712 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:29,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T18:14:29,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595329849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595329850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595329850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:29,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-07T18:14:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:29,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:29,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:29,883 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412072f767903a9e94aacab9246bf3c246954_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072f767903a9e94aacab9246bf3c246954_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:29,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/0411c3afbc4644628733ec48e6fbaefa, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:29,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/0411c3afbc4644628733ec48e6fbaefa is 175, key is test_row_0/A:col10/1733595268292/Put/seqid=0 2024-12-07T18:14:29,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742042_1218 (size=39949) 2024-12-07T18:14:29,937 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/0411c3afbc4644628733ec48e6fbaefa 2024-12-07T18:14:29,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/890c0a330ff2483288ac8995fd5cab12 is 50, key is test_row_0/B:col10/1733595268292/Put/seqid=0 2024-12-07T18:14:30,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742043_1219 (size=12301) 2024-12-07T18:14:30,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/890c0a330ff2483288ac8995fd5cab12 2024-12-07T18:14:30,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-07T18:14:30,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:30,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T18:14:30,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/3b839c6df98f48b7b00f1c07b92527dd is 50, key is test_row_0/C:col10/1733595268292/Put/seqid=0 2024-12-07T18:14:30,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742044_1220 (size=12301) 2024-12-07T18:14:30,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595330158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:30,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595330159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:30,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595330159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:30,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-07T18:14:30,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:30,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-07T18:14:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T18:14:30,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/3b839c6df98f48b7b00f1c07b92527dd 2024-12-07T18:14:30,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-07T18:14:30,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:30,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:30,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/0411c3afbc4644628733ec48e6fbaefa as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0411c3afbc4644628733ec48e6fbaefa 2024-12-07T18:14:30,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0411c3afbc4644628733ec48e6fbaefa, entries=200, sequenceid=387, filesize=39.0 K 2024-12-07T18:14:30,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/890c0a330ff2483288ac8995fd5cab12 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/890c0a330ff2483288ac8995fd5cab12 2024-12-07T18:14:30,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/890c0a330ff2483288ac8995fd5cab12, entries=150, sequenceid=387, filesize=12.0 K 2024-12-07T18:14:30,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/3b839c6df98f48b7b00f1c07b92527dd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3b839c6df98f48b7b00f1c07b92527dd 2024-12-07T18:14:30,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3b839c6df98f48b7b00f1c07b92527dd, entries=150, sequenceid=387, filesize=12.0 K 2024-12-07T18:14:30,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 3d520264937e773a7192a548dec4f0b4 in 1083ms, sequenceid=387, compaction requested=true 2024-12-07T18:14:30,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:30,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:30,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:30,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:30,523 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:30,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:30,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:30,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-07T18:14:30,524 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:30,525 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:30,525 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/A is initiating minor compaction (all files) 2024-12-07T18:14:30,525 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:30,525 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/A in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,525 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/B is initiating minor compaction (all files) 2024-12-07T18:14:30,525 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/B in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,525 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/9a19131fecc1447598a1e8da24146367, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bc0b50f2c28649e197458ce85d2e52a5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0411c3afbc4644628733ec48e6fbaefa] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=100.7 K 2024-12-07T18:14:30,525 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6a706aba738d4c25b55003b9647de391, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ff27580b25154594bad26b91107db56a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/890c0a330ff2483288ac8995fd5cab12] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=36.7 K 2024-12-07T18:14:30,525 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,525 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/9a19131fecc1447598a1e8da24146367, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bc0b50f2c28649e197458ce85d2e52a5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0411c3afbc4644628733ec48e6fbaefa] 2024-12-07T18:14:30,526 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a19131fecc1447598a1e8da24146367, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733595266441 2024-12-07T18:14:30,526 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a706aba738d4c25b55003b9647de391, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733595266441 2024-12-07T18:14:30,526 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff27580b25154594bad26b91107db56a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733595267119 2024-12-07T18:14:30,526 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting bc0b50f2c28649e197458ce85d2e52a5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733595267119 2024-12-07T18:14:30,527 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 0411c3afbc4644628733ec48e6fbaefa, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733595268292 2024-12-07T18:14:30,527 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 890c0a330ff2483288ac8995fd5cab12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733595268292 2024-12-07T18:14:30,556 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#B#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:30,557 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/cbc4f1aec4ca4419a60636cc300af10f is 50, key is test_row_0/B:col10/1733595268292/Put/seqid=0 2024-12-07T18:14:30,563 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:30,591 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207618fe05152ea4e8596f3beaedbb57b67_3d520264937e773a7192a548dec4f0b4 store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:30,593 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207618fe05152ea4e8596f3beaedbb57b67_3d520264937e773a7192a548dec4f0b4, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:30,594 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207618fe05152ea4e8596f3beaedbb57b67_3d520264937e773a7192a548dec4f0b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:30,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742045_1221 (size=13051) 2024-12-07T18:14:30,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-07T18:14:30,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,643 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T18:14:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:30,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:30,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742046_1222 (size=4469) 2024-12-07T18:14:30,650 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#A#compaction#184 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:30,651 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/ade7ca11688a4b939f043100b0590b9a is 175, key is test_row_0/A:col10/1733595268292/Put/seqid=0 2024-12-07T18:14:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412074ff20ec2ee8446b984ffa3836d5ed041_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595269514/Put/seqid=0 2024-12-07T18:14:30,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:30,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742047_1223 (size=32005) 2024-12-07T18:14:30,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595330697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:30,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742048_1224 (size=12454) 2024-12-07T18:14:30,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595330701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:30,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:30,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595330703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:30,728 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/ade7ca11688a4b939f043100b0590b9a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/ade7ca11688a4b939f043100b0590b9a 2024-12-07T18:14:30,729 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412074ff20ec2ee8446b984ffa3836d5ed041_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412074ff20ec2ee8446b984ffa3836d5ed041_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:30,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/d35c8f8e02234ac2ae8f46d3fe1dc739, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:30,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/d35c8f8e02234ac2ae8f46d3fe1dc739 is 175, key is test_row_0/A:col10/1733595269514/Put/seqid=0 2024-12-07T18:14:30,737 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/A of 3d520264937e773a7192a548dec4f0b4 into ade7ca11688a4b939f043100b0590b9a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:30,737 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:30,737 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/A, priority=13, startTime=1733595270523; duration=0sec 2024-12-07T18:14:30,737 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:30,737 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:A 2024-12-07T18:14:30,738 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:30,739 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:30,739 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/C is initiating minor compaction (all files) 2024-12-07T18:14:30,739 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/C in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:30,739 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/42eae6755dd8454e91a385814c63c498, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c40df5e8ce9a41e19c1540be3abb9f0d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3b839c6df98f48b7b00f1c07b92527dd] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=36.7 K 2024-12-07T18:14:30,739 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 42eae6755dd8454e91a385814c63c498, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733595266441 2024-12-07T18:14:30,740 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting c40df5e8ce9a41e19c1540be3abb9f0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1733595267119 2024-12-07T18:14:30,740 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b839c6df98f48b7b00f1c07b92527dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733595268292 2024-12-07T18:14:30,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742049_1225 (size=31255) 2024-12-07T18:14:30,771 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:30,771 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/c9b7869d45b841ee854f94a02db6b63e is 50, key is test_row_0/C:col10/1733595268292/Put/seqid=0 2024-12-07T18:14:30,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:30,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595330808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:30,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742050_1226 (size=13051) 2024-12-07T18:14:30,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595330819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:30,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595330828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:30,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T18:14:31,019 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/cbc4f1aec4ca4419a60636cc300af10f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cbc4f1aec4ca4419a60636cc300af10f 2024-12-07T18:14:31,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:31,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595331018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:31,026 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/B of 3d520264937e773a7192a548dec4f0b4 into cbc4f1aec4ca4419a60636cc300af10f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:31,026 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:31,026 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/B, priority=13, startTime=1733595270523; duration=0sec 2024-12-07T18:14:31,026 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:31,026 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:B 2024-12-07T18:14:31,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595331029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:31,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595331038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:31,168 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=407, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/d35c8f8e02234ac2ae8f46d3fe1dc739 2024-12-07T18:14:31,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/397be20819db4150826f1293a6987cf6 is 50, key is test_row_0/B:col10/1733595269514/Put/seqid=0 2024-12-07T18:14:31,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742051_1227 (size=12301) 2024-12-07T18:14:31,235 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/c9b7869d45b841ee854f94a02db6b63e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c9b7869d45b841ee854f94a02db6b63e 2024-12-07T18:14:31,243 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/C of 3d520264937e773a7192a548dec4f0b4 into c9b7869d45b841ee854f94a02db6b63e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:31,243 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:31,243 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/C, priority=13, startTime=1733595270523; duration=0sec 2024-12-07T18:14:31,243 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:31,243 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:C 2024-12-07T18:14:31,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:31,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595331328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:31,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:31,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595331338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:31,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595331349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:31,601 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/397be20819db4150826f1293a6987cf6 2024-12-07T18:14:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/09e98f4a45054221aca9770d342ef053 is 50, key is test_row_0/C:col10/1733595269514/Put/seqid=0 2024-12-07T18:14:31,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742052_1228 (size=12301) 2024-12-07T18:14:31,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595331838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:31,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595331849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:31,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T18:14:31,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595331858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:31,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:31,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:32,015 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/09e98f4a45054221aca9770d342ef053 2024-12-07T18:14:32,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/d35c8f8e02234ac2ae8f46d3fe1dc739 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/d35c8f8e02234ac2ae8f46d3fe1dc739 2024-12-07T18:14:32,024 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/d35c8f8e02234ac2ae8f46d3fe1dc739, entries=150, sequenceid=407, filesize=30.5 K 2024-12-07T18:14:32,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/397be20819db4150826f1293a6987cf6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/397be20819db4150826f1293a6987cf6 2024-12-07T18:14:32,029 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/397be20819db4150826f1293a6987cf6, entries=150, sequenceid=407, filesize=12.0 K 2024-12-07T18:14:32,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/09e98f4a45054221aca9770d342ef053 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/09e98f4a45054221aca9770d342ef053 2024-12-07T18:14:32,039 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/09e98f4a45054221aca9770d342ef053, entries=150, sequenceid=407, filesize=12.0 K 2024-12-07T18:14:32,040 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3d520264937e773a7192a548dec4f0b4 in 1397ms, sequenceid=407, compaction requested=false 2024-12-07T18:14:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:32,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-07T18:14:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-07T18:14:32,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-07T18:14:32,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3320 sec 2024-12-07T18:14:32,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.3420 sec 2024-12-07T18:14:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:32,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T18:14:32,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:32,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:32,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:32,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:32,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:32,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:32,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120702741de4e3e5434098c2da6672c3deac_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595270699/Put/seqid=0 2024-12-07T18:14:32,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742053_1229 (size=14994) 2024-12-07T18:14:32,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595332901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:32,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595332901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:32,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595332901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:32,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:32,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:32,905 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595333008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595333009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:33,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595333009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,015 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595333218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:33,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595333218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595333218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,279 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:33,284 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120702741de4e3e5434098c2da6672c3deac_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120702741de4e3e5434098c2da6672c3deac_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:33,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/25af058948f346f6aa66362f39b31906, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:33,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/25af058948f346f6aa66362f39b31906 is 175, key is test_row_0/A:col10/1733595270699/Put/seqid=0 2024-12-07T18:14:33,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742054_1230 (size=39949) 2024-12-07T18:14:33,325 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/25af058948f346f6aa66362f39b31906 2024-12-07T18:14:33,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/7f1129f044ba450fb36bc15b5e626100 is 50, key is test_row_0/B:col10/1733595270699/Put/seqid=0 2024-12-07T18:14:33,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742055_1231 (size=12301) 2024-12-07T18:14:33,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595333528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595333528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595333529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:33,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39878 deadline: 1733595333568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:33,570 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 3d520264937e773a7192a548dec4f0b4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:14:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:33,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:33,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/7f1129f044ba450fb36bc15b5e626100 2024-12-07T18:14:33,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/6a00d5a86f0d42abbadbddacab8106b7 is 50, key is test_row_0/C:col10/1733595270699/Put/seqid=0 2024-12-07T18:14:33,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742056_1232 (size=12301) 2024-12-07T18:14:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-07T18:14:33,868 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:33,869 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-07T18:14:33,871 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T18:14:33,872 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:33,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T18:14:34,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T18:14:34,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:34,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:34,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:34,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:34,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:34,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595334038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:34,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595334038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:34,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:34,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595334048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:34,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:34,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/6a00d5a86f0d42abbadbddacab8106b7 2024-12-07T18:14:34,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/25af058948f346f6aa66362f39b31906 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/25af058948f346f6aa66362f39b31906 2024-12-07T18:14:34,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/25af058948f346f6aa66362f39b31906, entries=200, sequenceid=423, filesize=39.0 K 2024-12-07T18:14:34,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/7f1129f044ba450fb36bc15b5e626100 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7f1129f044ba450fb36bc15b5e626100 2024-12-07T18:14:34,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T18:14:34,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:34,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:34,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:34,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:34,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7f1129f044ba450fb36bc15b5e626100, entries=150, sequenceid=423, filesize=12.0 K 2024-12-07T18:14:34,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/6a00d5a86f0d42abbadbddacab8106b7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/6a00d5a86f0d42abbadbddacab8106b7 2024-12-07T18:14:34,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/6a00d5a86f0d42abbadbddacab8106b7, entries=150, sequenceid=423, filesize=12.0 K 2024-12-07T18:14:34,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 3d520264937e773a7192a548dec4f0b4 in 1327ms, sequenceid=423, compaction requested=true 2024-12-07T18:14:34,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T18:14:34,189 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:34,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:34,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:34,189 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:34,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:34,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:34,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:34,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:34,191 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:34,191 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/A is initiating minor compaction (all files) 2024-12-07T18:14:34,191 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/A in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:34,191 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/ade7ca11688a4b939f043100b0590b9a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/d35c8f8e02234ac2ae8f46d3fe1dc739, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/25af058948f346f6aa66362f39b31906] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=100.8 K 2024-12-07T18:14:34,191 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:34,191 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/ade7ca11688a4b939f043100b0590b9a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/d35c8f8e02234ac2ae8f46d3fe1dc739, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/25af058948f346f6aa66362f39b31906] 2024-12-07T18:14:34,191 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:34,191 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/B is initiating minor compaction (all files) 2024-12-07T18:14:34,191 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/B in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:34,191 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting ade7ca11688a4b939f043100b0590b9a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733595268292 2024-12-07T18:14:34,192 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cbc4f1aec4ca4419a60636cc300af10f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/397be20819db4150826f1293a6987cf6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7f1129f044ba450fb36bc15b5e626100] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=36.8 K 2024-12-07T18:14:34,192 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting d35c8f8e02234ac2ae8f46d3fe1dc739, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733595269514 2024-12-07T18:14:34,192 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting cbc4f1aec4ca4419a60636cc300af10f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733595268292 2024-12-07T18:14:34,192 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25af058948f346f6aa66362f39b31906, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733595270690 2024-12-07T18:14:34,193 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 397be20819db4150826f1293a6987cf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733595269514 2024-12-07T18:14:34,193 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f1129f044ba450fb36bc15b5e626100, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733595270699 2024-12-07T18:14:34,212 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:34,217 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#B#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:34,218 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/cb2c50901c614fa889877b7ed846e3cb is 50, key is test_row_0/B:col10/1733595270699/Put/seqid=0 2024-12-07T18:14:34,219 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412075e1409e59b134fcc83811dc91a61ff58_3d520264937e773a7192a548dec4f0b4 store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:34,222 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412075e1409e59b134fcc83811dc91a61ff58_3d520264937e773a7192a548dec4f0b4, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:34,222 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412075e1409e59b134fcc83811dc91a61ff58_3d520264937e773a7192a548dec4f0b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:34,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742057_1233 (size=4469) 2024-12-07T18:14:34,248 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#A#compaction#192 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:34,249 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/e9ebca0b08b54c90804f509a66825a42 is 175, key is test_row_0/A:col10/1733595270699/Put/seqid=0 2024-12-07T18:14:34,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742058_1234 (size=13153) 2024-12-07T18:14:34,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742059_1235 (size=32107) 2024-12-07T18:14:34,277 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/e9ebca0b08b54c90804f509a66825a42 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e9ebca0b08b54c90804f509a66825a42 2024-12-07T18:14:34,285 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/A of 3d520264937e773a7192a548dec4f0b4 into e9ebca0b08b54c90804f509a66825a42(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:34,285 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:34,285 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/A, priority=13, startTime=1733595274189; duration=0sec 2024-12-07T18:14:34,285 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:34,285 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:A 2024-12-07T18:14:34,285 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:34,287 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:34,287 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/C is initiating minor compaction (all files) 2024-12-07T18:14:34,288 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/C in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:34,288 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c9b7869d45b841ee854f94a02db6b63e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/09e98f4a45054221aca9770d342ef053, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/6a00d5a86f0d42abbadbddacab8106b7] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=36.8 K 2024-12-07T18:14:34,289 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9b7869d45b841ee854f94a02db6b63e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733595268292 2024-12-07T18:14:34,289 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e98f4a45054221aca9770d342ef053, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1733595269514 2024-12-07T18:14:34,290 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a00d5a86f0d42abbadbddacab8106b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733595270699 2024-12-07T18:14:34,297 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#C#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:34,298 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/959dc380bb2841a98c5841b2f2dd45d9 is 50, key is test_row_0/C:col10/1733595270699/Put/seqid=0 2024-12-07T18:14:34,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742060_1236 (size=13153) 2024-12-07T18:14:34,323 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/959dc380bb2841a98c5841b2f2dd45d9 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/959dc380bb2841a98c5841b2f2dd45d9 2024-12-07T18:14:34,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-07T18:14:34,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:34,333 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T18:14:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:34,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:34,345 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/C of 3d520264937e773a7192a548dec4f0b4 into 959dc380bb2841a98c5841b2f2dd45d9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:34,345 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:34,345 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/C, priority=13, startTime=1733595274189; duration=0sec 2024-12-07T18:14:34,345 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:34,345 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:C 2024-12-07T18:14:34,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120750a87d87ca29494b951d17d85b3105de_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595272900/Put/seqid=0 2024-12-07T18:14:34,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742061_1237 (size=12454) 2024-12-07T18:14:34,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T18:14:34,662 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/cb2c50901c614fa889877b7ed846e3cb as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb2c50901c614fa889877b7ed846e3cb 2024-12-07T18:14:34,667 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/B of 3d520264937e773a7192a548dec4f0b4 into cb2c50901c614fa889877b7ed846e3cb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:34,667 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:34,667 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/B, priority=13, startTime=1733595274189; duration=0sec 2024-12-07T18:14:34,667 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:34,667 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:B 2024-12-07T18:14:34,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:34,801 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120750a87d87ca29494b951d17d85b3105de_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120750a87d87ca29494b951d17d85b3105de_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:34,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/0907e484b46346c39848a157c6e9e5cc, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:34,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/0907e484b46346c39848a157c6e9e5cc is 175, key is test_row_0/A:col10/1733595272900/Put/seqid=0 2024-12-07T18:14:34,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742062_1238 (size=31255) 2024-12-07T18:14:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T18:14:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:35,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. as already flushing 2024-12-07T18:14:35,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595335067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595335068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595335068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595335178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595335178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595335178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,214 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=444, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/0907e484b46346c39848a157c6e9e5cc 2024-12-07T18:14:35,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/355537cf4f5842b8a8aa6991e2e93075 is 50, key is test_row_0/B:col10/1733595272900/Put/seqid=0 2024-12-07T18:14:35,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742063_1239 (size=12301) 2024-12-07T18:14:35,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595335388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595335388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595335388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,434 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T18:14:35,660 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/355537cf4f5842b8a8aa6991e2e93075 2024-12-07T18:14:35,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/e69c2fc0d268450cafb121dffa5be432 is 50, key is test_row_0/C:col10/1733595272900/Put/seqid=0 2024-12-07T18:14:35,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742064_1240 (size=12301) 2024-12-07T18:14:35,686 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/e69c2fc0d268450cafb121dffa5be432 2024-12-07T18:14:35,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39848 deadline: 1733595335698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39836 deadline: 1733595335698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39872 deadline: 1733595335698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:35,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:14:35,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/0907e484b46346c39848a157c6e9e5cc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0907e484b46346c39848a157c6e9e5cc 2024-12-07T18:14:35,710 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0907e484b46346c39848a157c6e9e5cc, entries=150, sequenceid=444, filesize=30.5 K 2024-12-07T18:14:35,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/355537cf4f5842b8a8aa6991e2e93075 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/355537cf4f5842b8a8aa6991e2e93075 2024-12-07T18:14:35,715 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/355537cf4f5842b8a8aa6991e2e93075, entries=150, sequenceid=444, filesize=12.0 K 2024-12-07T18:14:35,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/e69c2fc0d268450cafb121dffa5be432 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/e69c2fc0d268450cafb121dffa5be432 2024-12-07T18:14:35,720 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/e69c2fc0d268450cafb121dffa5be432, entries=150, sequenceid=444, filesize=12.0 K 2024-12-07T18:14:35,721 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3d520264937e773a7192a548dec4f0b4 in 1388ms, sequenceid=444, compaction requested=false 2024-12-07T18:14:35,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:35,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:35,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-07T18:14:35,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-07T18:14:35,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-07T18:14:35,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8500 sec 2024-12-07T18:14:35,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8580 sec 2024-12-07T18:14:35,796 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-12-07T18:14:35,796 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:35,796 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:35,796 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:35,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:35,798 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-12-07T18:14:35,798 INFO [Thread-679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-679. 2024-12-07T18:14:35,798 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:35,798 DEBUG [Thread-679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:35,798 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:35,798 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:35,798 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:35,798 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:35,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:35,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:35,799 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-12-07T18:14:35,799 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:35,799 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:35,800 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:35,800 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-07T18:14:36,019 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:36,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T18:14:36,211 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-12-07T18:14:36,211 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:36,211 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:36,211 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:36,211 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:36,218 INFO [Thread-668 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-668. 2024-12-07T18:14:36,218 DEBUG [Thread-668 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:36,218 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:36,218 INFO [Thread-666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-666. 2024-12-07T18:14:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:36,218 DEBUG [Thread-666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:36,218 DEBUG [Thread-668 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:36,218 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:36,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:36,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:36,218 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:36,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:36,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412077a429fa478524ef9b9c6639a9b68780f_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595276209/Put/seqid=0 2024-12-07T18:14:36,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742065_1241 (size=12454) 2024-12-07T18:14:36,520 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-12-07T18:14:36,520 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:36,520 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:36,520 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:36,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:36,650 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:36,654 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412077a429fa478524ef9b9c6639a9b68780f_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077a429fa478524ef9b9c6639a9b68780f_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:36,655 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/35dec8b7ce26421db5ea90af792e018e, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:36,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/35dec8b7ce26421db5ea90af792e018e is 175, key is test_row_0/A:col10/1733595276209/Put/seqid=0 2024-12-07T18:14:36,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742066_1242 (size=31255) 2024-12-07T18:14:37,060 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=461, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/35dec8b7ce26421db5ea90af792e018e 2024-12-07T18:14:37,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/6ae83492b84047b79c06291fb8d7c5da is 50, key is test_row_0/B:col10/1733595276209/Put/seqid=0 2024-12-07T18:14:37,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742067_1243 (size=12301) 2024-12-07T18:14:37,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/6ae83492b84047b79c06291fb8d7c5da 2024-12-07T18:14:37,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/628781d307e2414988da4ac08907fdc4 is 50, key is test_row_0/C:col10/1733595276209/Put/seqid=0 2024-12-07T18:14:37,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742068_1244 (size=12301) 2024-12-07T18:14:37,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/628781d307e2414988da4ac08907fdc4 2024-12-07T18:14:37,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/35dec8b7ce26421db5ea90af792e018e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/35dec8b7ce26421db5ea90af792e018e 2024-12-07T18:14:37,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/35dec8b7ce26421db5ea90af792e018e, entries=150, sequenceid=461, filesize=30.5 K 2024-12-07T18:14:37,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/6ae83492b84047b79c06291fb8d7c5da as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6ae83492b84047b79c06291fb8d7c5da 2024-12-07T18:14:37,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6ae83492b84047b79c06291fb8d7c5da, entries=150, sequenceid=461, filesize=12.0 K 2024-12-07T18:14:37,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/628781d307e2414988da4ac08907fdc4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/628781d307e2414988da4ac08907fdc4 2024-12-07T18:14:37,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/628781d307e2414988da4ac08907fdc4, entries=150, sequenceid=461, filesize=12.0 K 2024-12-07T18:14:37,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=6.71 KB/6870 for 3d520264937e773a7192a548dec4f0b4 in 1704ms, sequenceid=461, compaction requested=true 2024-12-07T18:14:37,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:37,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:37,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:37,915 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:37,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:37,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:37,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3d520264937e773a7192a548dec4f0b4:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:37,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:37,916 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:37,917 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:37,917 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:37,917 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/A is initiating minor compaction (all files) 2024-12-07T18:14:37,917 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/B is initiating minor compaction (all files) 2024-12-07T18:14:37,917 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/B in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:37,917 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/A in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:37,917 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb2c50901c614fa889877b7ed846e3cb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/355537cf4f5842b8a8aa6991e2e93075, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6ae83492b84047b79c06291fb8d7c5da] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=36.9 K 2024-12-07T18:14:37,917 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e9ebca0b08b54c90804f509a66825a42, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0907e484b46346c39848a157c6e9e5cc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/35dec8b7ce26421db5ea90af792e018e] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=92.4 K 2024-12-07T18:14:37,917 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:37,917 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e9ebca0b08b54c90804f509a66825a42, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0907e484b46346c39848a157c6e9e5cc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/35dec8b7ce26421db5ea90af792e018e] 2024-12-07T18:14:37,918 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2c50901c614fa889877b7ed846e3cb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733595270699 2024-12-07T18:14:37,918 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9ebca0b08b54c90804f509a66825a42, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733595270699 2024-12-07T18:14:37,918 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 355537cf4f5842b8a8aa6991e2e93075, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733595272898 2024-12-07T18:14:37,918 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0907e484b46346c39848a157c6e9e5cc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733595272898 2024-12-07T18:14:37,918 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ae83492b84047b79c06291fb8d7c5da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733595275059 2024-12-07T18:14:37,918 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35dec8b7ce26421db5ea90af792e018e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733595275059 2024-12-07T18:14:37,931 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#B#compaction#201 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:37,932 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/2105008ffcb441359fb3929344ae4b95 is 50, key is test_row_0/B:col10/1733595276209/Put/seqid=0 2024-12-07T18:14:37,937 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:37,944 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207af84729a16ed4d24912843cf6d4424d7_3d520264937e773a7192a548dec4f0b4 store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:37,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742069_1245 (size=13255) 2024-12-07T18:14:37,975 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207af84729a16ed4d24912843cf6d4424d7_3d520264937e773a7192a548dec4f0b4, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:37,975 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207af84729a16ed4d24912843cf6d4424d7_3d520264937e773a7192a548dec4f0b4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:37,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742070_1246 (size=4469) 2024-12-07T18:14:37,989 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#A#compaction#202 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:37,990 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/6470651c1f9b4f029ccdd406013d6a0e is 175, key is test_row_0/A:col10/1733595276209/Put/seqid=0 2024-12-07T18:14:38,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742071_1247 (size=32209) 2024-12-07T18:14:38,380 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/2105008ffcb441359fb3929344ae4b95 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/2105008ffcb441359fb3929344ae4b95 2024-12-07T18:14:38,385 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/B of 3d520264937e773a7192a548dec4f0b4 into 2105008ffcb441359fb3929344ae4b95(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:38,386 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:38,386 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/B, priority=13, startTime=1733595277915; duration=0sec 2024-12-07T18:14:38,386 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:38,386 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:B 2024-12-07T18:14:38,386 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:38,387 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:38,387 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 3d520264937e773a7192a548dec4f0b4/C is initiating minor compaction (all files) 2024-12-07T18:14:38,387 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3d520264937e773a7192a548dec4f0b4/C in TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:38,387 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/959dc380bb2841a98c5841b2f2dd45d9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/e69c2fc0d268450cafb121dffa5be432, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/628781d307e2414988da4ac08907fdc4] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp, totalSize=36.9 K 2024-12-07T18:14:38,387 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 959dc380bb2841a98c5841b2f2dd45d9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1733595270699 2024-12-07T18:14:38,388 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e69c2fc0d268450cafb121dffa5be432, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733595272898 2024-12-07T18:14:38,388 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 628781d307e2414988da4ac08907fdc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=461, earliestPutTs=1733595275059 2024-12-07T18:14:38,395 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3d520264937e773a7192a548dec4f0b4#C#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:38,396 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/36e14c4483714d58b09211c4564be8c7 is 50, key is test_row_0/C:col10/1733595276209/Put/seqid=0 2024-12-07T18:14:38,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742072_1248 (size=13255) 2024-12-07T18:14:38,408 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/6470651c1f9b4f029ccdd406013d6a0e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/6470651c1f9b4f029ccdd406013d6a0e 2024-12-07T18:14:38,414 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/A of 3d520264937e773a7192a548dec4f0b4 into 6470651c1f9b4f029ccdd406013d6a0e(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:38,414 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:38,414 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/A, priority=13, startTime=1733595277915; duration=0sec 2024-12-07T18:14:38,414 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:38,414 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:A 2024-12-07T18:14:38,808 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/36e14c4483714d58b09211c4564be8c7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/36e14c4483714d58b09211c4564be8c7 2024-12-07T18:14:38,813 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3d520264937e773a7192a548dec4f0b4/C of 3d520264937e773a7192a548dec4f0b4 into 36e14c4483714d58b09211c4564be8c7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:38,813 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3d520264937e773a7192a548dec4f0b4: 2024-12-07T18:14:38,813 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., storeName=3d520264937e773a7192a548dec4f0b4/C, priority=13, startTime=1733595277916; duration=0sec 2024-12-07T18:14:38,813 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:38,813 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3d520264937e773a7192a548dec4f0b4:C 2024-12-07T18:14:43,599 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-12-07T18:14:43,599 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:14:43,599 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:43,600 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:43,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 172 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2495 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2409 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3309 rows 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-12-07T18:14:43,600 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3305 rows 2024-12-07T18:14:43,600 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:14:43,600 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:14:43,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:43,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:43,600 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:14:43,601 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T18:14:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:43,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T18:14:43,604 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595283604"}]},"ts":"1733595283604"} 2024-12-07T18:14:43,605 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T18:14:43,605 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T18:14:43,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T18:14:43,607 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, UNASSIGN}] 2024-12-07T18:14:43,608 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, UNASSIGN 2024-12-07T18:14:43,608 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=3d520264937e773a7192a548dec4f0b4, regionState=CLOSING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:43,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, UNASSIGN because future has completed 2024-12-07T18:14:43,610 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T18:14:43,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:14:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T18:14:43,762 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 3d520264937e773a7192a548dec4f0b4, disabling compactions & flushes 2024-12-07T18:14:43,762 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. after waiting 0 ms 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:43,762 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 3d520264937e773a7192a548dec4f0b4 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=A 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=B 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3d520264937e773a7192a548dec4f0b4, store=C 2024-12-07T18:14:43,762 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:43,769 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412077367c00d166c4e64a3c330098d14acf4_3d520264937e773a7192a548dec4f0b4 is 50, key is test_row_0/A:col10/1733595283598/Put/seqid=0 2024-12-07T18:14:43,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742073_1249 (size=7374) 2024-12-07T18:14:43,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T18:14:44,173 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:44,178 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412077367c00d166c4e64a3c330098d14acf4_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077367c00d166c4e64a3c330098d14acf4_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:44,179 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/32054cd5e0b84f30ace0caced10355cc, store: [table=TestAcidGuarantees family=A region=3d520264937e773a7192a548dec4f0b4] 2024-12-07T18:14:44,180 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/32054cd5e0b84f30ace0caced10355cc is 175, key is test_row_0/A:col10/1733595283598/Put/seqid=0 2024-12-07T18:14:44,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742074_1250 (size=13865) 2024-12-07T18:14:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T18:14:44,584 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=469, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/32054cd5e0b84f30ace0caced10355cc 2024-12-07T18:14:44,592 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/8dde49d6c1224b05bf8853f7259f0dc8 is 50, key is test_row_0/B:col10/1733595283598/Put/seqid=0 2024-12-07T18:14:44,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742075_1251 (size=7415) 2024-12-07T18:14:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T18:14:44,996 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/8dde49d6c1224b05bf8853f7259f0dc8 2024-12-07T18:14:45,003 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/792ce5139f864d739e817879f1574a51 is 50, key is test_row_0/C:col10/1733595283598/Put/seqid=0 2024-12-07T18:14:45,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742076_1252 (size=7415) 2024-12-07T18:14:45,407 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/792ce5139f864d739e817879f1574a51 2024-12-07T18:14:45,412 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/A/32054cd5e0b84f30ace0caced10355cc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/32054cd5e0b84f30ace0caced10355cc 2024-12-07T18:14:45,415 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/32054cd5e0b84f30ace0caced10355cc, entries=50, sequenceid=469, filesize=13.5 K 2024-12-07T18:14:45,416 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/B/8dde49d6c1224b05bf8853f7259f0dc8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/8dde49d6c1224b05bf8853f7259f0dc8 2024-12-07T18:14:45,420 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/8dde49d6c1224b05bf8853f7259f0dc8, entries=50, sequenceid=469, filesize=7.2 K 2024-12-07T18:14:45,421 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/.tmp/C/792ce5139f864d739e817879f1574a51 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/792ce5139f864d739e817879f1574a51 2024-12-07T18:14:45,425 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/792ce5139f864d739e817879f1574a51, entries=50, sequenceid=469, filesize=7.2 K 2024-12-07T18:14:45,425 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 3d520264937e773a7192a548dec4f0b4 in 1663ms, sequenceid=469, compaction requested=false 2024-12-07T18:14:45,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/3395dd52baa640e6aa23d6bec190937b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/794c223d7fed486da9b17495fed93034, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/168b986cc5724e5982ee5116b2736907, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/aba59a87d9f842faa7c2d509c800d0db, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/dbeb9f9749ac4d9e91afe5c4f3374de3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/fbe7496f1b32428d978cd11741c36cb2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/da2ec569a4e14e4aba09b9e8618218ac, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c50b29ba59824356bc3666d28f3a1c4f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/11d8b7e8bc27433ea63178e14ae62904, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e8d903087ce848d8bd9e1872f95627ea, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c00f7e82c92a45c89dbef9778a2c8b64, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/57a8b8103ef7477290a9d841b78845c7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf392cf45adf4eaf9eeb26011384ac40, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/7136274d3a9741de956de0f110a5d7cd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf676abbed7b40d583f29da54e0c5641, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c2eca407342948628bf7f2080a0258d6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/20f52886415d4b5da708a3f625d7be52, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/adf12d7ade1140e3bc4f8120d2e1c9c8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/9a19131fecc1447598a1e8da24146367, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bc0b50f2c28649e197458ce85d2e52a5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0411c3afbc4644628733ec48e6fbaefa, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/ade7ca11688a4b939f043100b0590b9a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/d35c8f8e02234ac2ae8f46d3fe1dc739, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/25af058948f346f6aa66362f39b31906, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e9ebca0b08b54c90804f509a66825a42, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0907e484b46346c39848a157c6e9e5cc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/35dec8b7ce26421db5ea90af792e018e] to archive 2024-12-07T18:14:45,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:14:45,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/3395dd52baa640e6aa23d6bec190937b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/3395dd52baa640e6aa23d6bec190937b 2024-12-07T18:14:45,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/794c223d7fed486da9b17495fed93034 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/794c223d7fed486da9b17495fed93034 2024-12-07T18:14:45,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/168b986cc5724e5982ee5116b2736907 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/168b986cc5724e5982ee5116b2736907 2024-12-07T18:14:45,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/aba59a87d9f842faa7c2d509c800d0db to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/aba59a87d9f842faa7c2d509c800d0db 2024-12-07T18:14:45,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/dbeb9f9749ac4d9e91afe5c4f3374de3 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/dbeb9f9749ac4d9e91afe5c4f3374de3 2024-12-07T18:14:45,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/fbe7496f1b32428d978cd11741c36cb2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/fbe7496f1b32428d978cd11741c36cb2 2024-12-07T18:14:45,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/da2ec569a4e14e4aba09b9e8618218ac to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/da2ec569a4e14e4aba09b9e8618218ac 2024-12-07T18:14:45,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c50b29ba59824356bc3666d28f3a1c4f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c50b29ba59824356bc3666d28f3a1c4f 2024-12-07T18:14:45,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/11d8b7e8bc27433ea63178e14ae62904 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/11d8b7e8bc27433ea63178e14ae62904 2024-12-07T18:14:45,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e8d903087ce848d8bd9e1872f95627ea to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e8d903087ce848d8bd9e1872f95627ea 2024-12-07T18:14:45,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c00f7e82c92a45c89dbef9778a2c8b64 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c00f7e82c92a45c89dbef9778a2c8b64 2024-12-07T18:14:45,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/57a8b8103ef7477290a9d841b78845c7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/57a8b8103ef7477290a9d841b78845c7 2024-12-07T18:14:45,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf392cf45adf4eaf9eeb26011384ac40 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf392cf45adf4eaf9eeb26011384ac40 2024-12-07T18:14:45,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/7136274d3a9741de956de0f110a5d7cd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/7136274d3a9741de956de0f110a5d7cd 2024-12-07T18:14:45,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf676abbed7b40d583f29da54e0c5641 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bf676abbed7b40d583f29da54e0c5641 2024-12-07T18:14:45,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c2eca407342948628bf7f2080a0258d6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/c2eca407342948628bf7f2080a0258d6 2024-12-07T18:14:45,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/20f52886415d4b5da708a3f625d7be52 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/20f52886415d4b5da708a3f625d7be52 2024-12-07T18:14:45,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/adf12d7ade1140e3bc4f8120d2e1c9c8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/adf12d7ade1140e3bc4f8120d2e1c9c8 2024-12-07T18:14:45,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/a670c34ce71a4bf1a8a1dc2ccc3aacb0 2024-12-07T18:14:45,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/9a19131fecc1447598a1e8da24146367 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/9a19131fecc1447598a1e8da24146367 2024-12-07T18:14:45,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bc0b50f2c28649e197458ce85d2e52a5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/bc0b50f2c28649e197458ce85d2e52a5 2024-12-07T18:14:45,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0411c3afbc4644628733ec48e6fbaefa to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0411c3afbc4644628733ec48e6fbaefa 2024-12-07T18:14:45,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/ade7ca11688a4b939f043100b0590b9a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/ade7ca11688a4b939f043100b0590b9a 2024-12-07T18:14:45,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/d35c8f8e02234ac2ae8f46d3fe1dc739 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/d35c8f8e02234ac2ae8f46d3fe1dc739 2024-12-07T18:14:45,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/25af058948f346f6aa66362f39b31906 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/25af058948f346f6aa66362f39b31906 2024-12-07T18:14:45,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e9ebca0b08b54c90804f509a66825a42 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/e9ebca0b08b54c90804f509a66825a42 2024-12-07T18:14:45,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0907e484b46346c39848a157c6e9e5cc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/0907e484b46346c39848a157c6e9e5cc 2024-12-07T18:14:45,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/35dec8b7ce26421db5ea90af792e018e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/35dec8b7ce26421db5ea90af792e018e 2024-12-07T18:14:45,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/b58f8a74c1b445a8907233fe3f0f44dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/9e3edf303e264978b7c3b16db54bfeb0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/5535a08381be49f4adaadaefba8fb3ba, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7d6e81a0fe144bb5816420d4a7551d2a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb7742c0a04c4f4e9ebd7641259eb1f3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/f8d4ee9731ac42318c5013b620c747f8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6387ef7662914401922350167c12f4b8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/a5a2bf63b63b4ab3bbcf3651b754500e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/c3b7375564ef44deb3ce57dd77688a4c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/3d4f5c5e4f21456abd7a5c1889035949, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ca7569ee992b4770bf2c684d57f231fa, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/082e275e401c48be8eb8383fa0ddcbf6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/48eb54076ab34ea58f906823803cdc07, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/26e0d524db044d29b3cde0fb62f03ccb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/161911049dac49ccae31f1c43b806c21, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/dab6e8439f3744c69977104d62d49382, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ad3647d05e2f4e25b69b85dde8bc97e1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ebfd4572a9674fcaa26f2f777ffe9c7d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6a706aba738d4c25b55003b9647de391, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/e0608918e0b244b5a9e3464eecaefbd6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ff27580b25154594bad26b91107db56a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cbc4f1aec4ca4419a60636cc300af10f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/890c0a330ff2483288ac8995fd5cab12, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/397be20819db4150826f1293a6987cf6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb2c50901c614fa889877b7ed846e3cb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7f1129f044ba450fb36bc15b5e626100, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/355537cf4f5842b8a8aa6991e2e93075, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6ae83492b84047b79c06291fb8d7c5da] to archive 2024-12-07T18:14:45,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:14:45,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/b58f8a74c1b445a8907233fe3f0f44dd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/b58f8a74c1b445a8907233fe3f0f44dd 2024-12-07T18:14:45,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/9e3edf303e264978b7c3b16db54bfeb0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/9e3edf303e264978b7c3b16db54bfeb0 2024-12-07T18:14:45,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/5535a08381be49f4adaadaefba8fb3ba to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/5535a08381be49f4adaadaefba8fb3ba 2024-12-07T18:14:45,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7d6e81a0fe144bb5816420d4a7551d2a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7d6e81a0fe144bb5816420d4a7551d2a 2024-12-07T18:14:45,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb7742c0a04c4f4e9ebd7641259eb1f3 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb7742c0a04c4f4e9ebd7641259eb1f3 2024-12-07T18:14:45,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/f8d4ee9731ac42318c5013b620c747f8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/f8d4ee9731ac42318c5013b620c747f8 2024-12-07T18:14:45,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6387ef7662914401922350167c12f4b8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6387ef7662914401922350167c12f4b8 2024-12-07T18:14:45,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/a5a2bf63b63b4ab3bbcf3651b754500e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/a5a2bf63b63b4ab3bbcf3651b754500e 2024-12-07T18:14:45,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/c3b7375564ef44deb3ce57dd77688a4c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/c3b7375564ef44deb3ce57dd77688a4c 2024-12-07T18:14:45,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/3d4f5c5e4f21456abd7a5c1889035949 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/3d4f5c5e4f21456abd7a5c1889035949 2024-12-07T18:14:45,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ca7569ee992b4770bf2c684d57f231fa to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ca7569ee992b4770bf2c684d57f231fa 2024-12-07T18:14:45,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/082e275e401c48be8eb8383fa0ddcbf6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/082e275e401c48be8eb8383fa0ddcbf6 2024-12-07T18:14:45,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/48eb54076ab34ea58f906823803cdc07 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/48eb54076ab34ea58f906823803cdc07 2024-12-07T18:14:45,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/26e0d524db044d29b3cde0fb62f03ccb to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/26e0d524db044d29b3cde0fb62f03ccb 2024-12-07T18:14:45,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/161911049dac49ccae31f1c43b806c21 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/161911049dac49ccae31f1c43b806c21 2024-12-07T18:14:45,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/dab6e8439f3744c69977104d62d49382 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/dab6e8439f3744c69977104d62d49382 2024-12-07T18:14:45,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ad3647d05e2f4e25b69b85dde8bc97e1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ad3647d05e2f4e25b69b85dde8bc97e1 2024-12-07T18:14:45,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ebfd4572a9674fcaa26f2f777ffe9c7d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ebfd4572a9674fcaa26f2f777ffe9c7d 2024-12-07T18:14:45,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6a706aba738d4c25b55003b9647de391 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6a706aba738d4c25b55003b9647de391 2024-12-07T18:14:45,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/e0608918e0b244b5a9e3464eecaefbd6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/e0608918e0b244b5a9e3464eecaefbd6 2024-12-07T18:14:45,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ff27580b25154594bad26b91107db56a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/ff27580b25154594bad26b91107db56a 2024-12-07T18:14:45,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cbc4f1aec4ca4419a60636cc300af10f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cbc4f1aec4ca4419a60636cc300af10f 2024-12-07T18:14:45,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/890c0a330ff2483288ac8995fd5cab12 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/890c0a330ff2483288ac8995fd5cab12 2024-12-07T18:14:45,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/397be20819db4150826f1293a6987cf6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/397be20819db4150826f1293a6987cf6 2024-12-07T18:14:45,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb2c50901c614fa889877b7ed846e3cb to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/cb2c50901c614fa889877b7ed846e3cb 2024-12-07T18:14:45,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7f1129f044ba450fb36bc15b5e626100 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/7f1129f044ba450fb36bc15b5e626100 2024-12-07T18:14:45,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/355537cf4f5842b8a8aa6991e2e93075 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/355537cf4f5842b8a8aa6991e2e93075 2024-12-07T18:14:45,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6ae83492b84047b79c06291fb8d7c5da to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/6ae83492b84047b79c06291fb8d7c5da 2024-12-07T18:14:45,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/0407c2f868b744cfb5143de1d2710eab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/7516bc76d0ba45df8007fcfc94d5067f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/d6dd97db0d2f4da89b75c19be121b861, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c918e01c3de64f3ea5cacbc1921efd73, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/623129636b2c4d54b649d41ff39cda02, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/df1bcf49fdb14895a6b3fa12b66d7b5e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/d3b01ca4b4ef492b9ebff6fc32fbbbb7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/f3bbe0cfeb0e48d88a27e6b2e6e03f20, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/a90fab7a5ffb43b5ada4f5741a771d5d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/ed867bef6ac644a393c3be9a0c62a9a5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/40b8f0a10add492ab3e986c47af5a92e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/b3ee8245bf684e0d8a19a73f196adb4e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/980d7128d15f45eaa6e7af61abfce758, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/43a0261a2fd043858cb30bf8e5ba7156, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/820772157654414fb0fe8bea80e71f3c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3d4b5fc6c74947788af579f9d062d7ab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/100fbd24e44448b8afd1216eaf2dc1c6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/831e5a0cdd7a4461b65f69ff6a3e818f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/42eae6755dd8454e91a385814c63c498, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/583b6073077648d6976bed26d345aa89, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c40df5e8ce9a41e19c1540be3abb9f0d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c9b7869d45b841ee854f94a02db6b63e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3b839c6df98f48b7b00f1c07b92527dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/09e98f4a45054221aca9770d342ef053, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/959dc380bb2841a98c5841b2f2dd45d9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/6a00d5a86f0d42abbadbddacab8106b7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/e69c2fc0d268450cafb121dffa5be432, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/628781d307e2414988da4ac08907fdc4] to archive 2024-12-07T18:14:45,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:14:45,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/0407c2f868b744cfb5143de1d2710eab to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/0407c2f868b744cfb5143de1d2710eab 2024-12-07T18:14:45,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/7516bc76d0ba45df8007fcfc94d5067f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/7516bc76d0ba45df8007fcfc94d5067f 2024-12-07T18:14:45,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/d6dd97db0d2f4da89b75c19be121b861 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/d6dd97db0d2f4da89b75c19be121b861 2024-12-07T18:14:45,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c918e01c3de64f3ea5cacbc1921efd73 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c918e01c3de64f3ea5cacbc1921efd73 2024-12-07T18:14:45,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/623129636b2c4d54b649d41ff39cda02 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/623129636b2c4d54b649d41ff39cda02 2024-12-07T18:14:45,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/df1bcf49fdb14895a6b3fa12b66d7b5e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/df1bcf49fdb14895a6b3fa12b66d7b5e 2024-12-07T18:14:45,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/d3b01ca4b4ef492b9ebff6fc32fbbbb7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/d3b01ca4b4ef492b9ebff6fc32fbbbb7 2024-12-07T18:14:45,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/f3bbe0cfeb0e48d88a27e6b2e6e03f20 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/f3bbe0cfeb0e48d88a27e6b2e6e03f20 2024-12-07T18:14:45,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/a90fab7a5ffb43b5ada4f5741a771d5d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/a90fab7a5ffb43b5ada4f5741a771d5d 2024-12-07T18:14:45,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/ed867bef6ac644a393c3be9a0c62a9a5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/ed867bef6ac644a393c3be9a0c62a9a5 2024-12-07T18:14:45,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/40b8f0a10add492ab3e986c47af5a92e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/40b8f0a10add492ab3e986c47af5a92e 2024-12-07T18:14:45,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/b3ee8245bf684e0d8a19a73f196adb4e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/b3ee8245bf684e0d8a19a73f196adb4e 2024-12-07T18:14:45,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/980d7128d15f45eaa6e7af61abfce758 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/980d7128d15f45eaa6e7af61abfce758 2024-12-07T18:14:45,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/43a0261a2fd043858cb30bf8e5ba7156 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/43a0261a2fd043858cb30bf8e5ba7156 2024-12-07T18:14:45,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/820772157654414fb0fe8bea80e71f3c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/820772157654414fb0fe8bea80e71f3c 2024-12-07T18:14:45,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3d4b5fc6c74947788af579f9d062d7ab to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3d4b5fc6c74947788af579f9d062d7ab 2024-12-07T18:14:45,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/100fbd24e44448b8afd1216eaf2dc1c6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/100fbd24e44448b8afd1216eaf2dc1c6 2024-12-07T18:14:45,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/831e5a0cdd7a4461b65f69ff6a3e818f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/831e5a0cdd7a4461b65f69ff6a3e818f 2024-12-07T18:14:45,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/42eae6755dd8454e91a385814c63c498 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/42eae6755dd8454e91a385814c63c498 2024-12-07T18:14:45,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/583b6073077648d6976bed26d345aa89 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/583b6073077648d6976bed26d345aa89 2024-12-07T18:14:45,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c40df5e8ce9a41e19c1540be3abb9f0d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c40df5e8ce9a41e19c1540be3abb9f0d 2024-12-07T18:14:45,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c9b7869d45b841ee854f94a02db6b63e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/c9b7869d45b841ee854f94a02db6b63e 2024-12-07T18:14:45,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3b839c6df98f48b7b00f1c07b92527dd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/3b839c6df98f48b7b00f1c07b92527dd 2024-12-07T18:14:45,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/09e98f4a45054221aca9770d342ef053 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/09e98f4a45054221aca9770d342ef053 2024-12-07T18:14:45,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/959dc380bb2841a98c5841b2f2dd45d9 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/959dc380bb2841a98c5841b2f2dd45d9 2024-12-07T18:14:45,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/6a00d5a86f0d42abbadbddacab8106b7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/6a00d5a86f0d42abbadbddacab8106b7 2024-12-07T18:14:45,526 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/e69c2fc0d268450cafb121dffa5be432 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/e69c2fc0d268450cafb121dffa5be432 2024-12-07T18:14:45,527 DEBUG [StoreCloser-TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/628781d307e2414988da4ac08907fdc4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/628781d307e2414988da4ac08907fdc4 2024-12-07T18:14:45,531 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/recovered.edits/472.seqid, newMaxSeqId=472, maxSeqId=4 2024-12-07T18:14:45,532 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. 2024-12-07T18:14:45,533 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 3d520264937e773a7192a548dec4f0b4: Waiting for close lock at 1733595283762Running coprocessor pre-close hooks at 1733595283762Disabling compacts and flushes for region at 1733595283762Disabling writes for close at 1733595283762Obtaining lock to block concurrent updates at 1733595283762Preparing flush snapshotting stores in 3d520264937e773a7192a548dec4f0b4 at 1733595283762Finished memstore snapshotting TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733595283762Flushing stores of TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4. at 1733595283763 (+1 ms)Flushing 3d520264937e773a7192a548dec4f0b4/A: creating writer at 1733595283763Flushing mob file 3d520264937e773a7192a548dec4f0b4/A: appending metadata at 1733595283769 (+6 ms)Flushing mob file 3d520264937e773a7192a548dec4f0b4/A: closing flushed file at 1733595283769Flushing 3d520264937e773a7192a548dec4f0b4/A: appending metadata at 1733595284179 (+410 ms)Flushing 3d520264937e773a7192a548dec4f0b4/A: closing flushed file at 1733595284179Flushing 3d520264937e773a7192a548dec4f0b4/B: creating writer at 1733595284588 (+409 ms)Flushing 3d520264937e773a7192a548dec4f0b4/B: appending metadata at 1733595284591 (+3 ms)Flushing 3d520264937e773a7192a548dec4f0b4/B: closing flushed file at 1733595284591Flushing 3d520264937e773a7192a548dec4f0b4/C: creating writer at 1733595285000 (+409 ms)Flushing 3d520264937e773a7192a548dec4f0b4/C: appending metadata at 1733595285003 (+3 ms)Flushing 3d520264937e773a7192a548dec4f0b4/C: closing flushed file at 1733595285003Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@364c7f24: reopening flushed file at 1733595285411 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cf0a61d: reopening flushed file at 1733595285415 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68d330fa: reopening flushed file at 1733595285420 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 3d520264937e773a7192a548dec4f0b4 in 1663ms, sequenceid=469, compaction requested=false at 1733595285425 (+5 ms)Writing region close event to WAL at 1733595285528 (+103 ms)Running coprocessor post-close hooks at 1733595285532 (+4 ms)Closed at 1733595285532 2024-12-07T18:14:45,534 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,535 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=3d520264937e773a7192a548dec4f0b4, regionState=CLOSED 2024-12-07T18:14:45,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:14:45,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-07T18:14:45,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 3d520264937e773a7192a548dec4f0b4, server=92acd7e2389b,33947,1733595217075 in 1.9270 sec 2024-12-07T18:14:45,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-07T18:14:45,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3d520264937e773a7192a548dec4f0b4, UNASSIGN in 1.9320 sec 2024-12-07T18:14:45,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-07T18:14:45,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9350 sec 2024-12-07T18:14:45,543 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595285543"}]},"ts":"1733595285543"} 2024-12-07T18:14:45,545 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T18:14:45,545 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T18:14:45,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9440 sec 2024-12-07T18:14:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-07T18:14:45,748 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:45,749 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T18:14:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:45,750 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-07T18:14:45,751 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:45,754 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,756 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/recovered.edits] 2024-12-07T18:14:45,758 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/32054cd5e0b84f30ace0caced10355cc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/32054cd5e0b84f30ace0caced10355cc 2024-12-07T18:14:45,760 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/6470651c1f9b4f029ccdd406013d6a0e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/A/6470651c1f9b4f029ccdd406013d6a0e 2024-12-07T18:14:45,761 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/2105008ffcb441359fb3929344ae4b95 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/2105008ffcb441359fb3929344ae4b95 2024-12-07T18:14:45,762 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/8dde49d6c1224b05bf8853f7259f0dc8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/B/8dde49d6c1224b05bf8853f7259f0dc8 2024-12-07T18:14:45,764 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/36e14c4483714d58b09211c4564be8c7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/36e14c4483714d58b09211c4564be8c7 2024-12-07T18:14:45,765 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/792ce5139f864d739e817879f1574a51 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/C/792ce5139f864d739e817879f1574a51 2024-12-07T18:14:45,767 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/recovered.edits/472.seqid to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4/recovered.edits/472.seqid 2024-12-07T18:14:45,768 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,768 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T18:14:45,768 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T18:14:45,769 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-07T18:14:45,772 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120701c94e934eb642108d69c3f9c4f77202_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120701c94e934eb642108d69c3f9c4f77202_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,773 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120702741de4e3e5434098c2da6672c3deac_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120702741de4e3e5434098c2da6672c3deac_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,774 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070cd3ac3455274fa1a6fc7ddc3cbbda7f_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412070cd3ac3455274fa1a6fc7ddc3cbbda7f_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,775 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071449e4f447d8478e8d3ae06eb2a81ebe_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071449e4f447d8478e8d3ae06eb2a81ebe_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,776 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072f767903a9e94aacab9246bf3c246954_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072f767903a9e94aacab9246bf3c246954_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,777 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073345adcc9701429cbf146b82d2d325a4_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073345adcc9701429cbf146b82d2d325a4_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,779 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412074ff20ec2ee8446b984ffa3836d5ed041_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412074ff20ec2ee8446b984ffa3836d5ed041_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,779 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120750a87d87ca29494b951d17d85b3105de_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120750a87d87ca29494b951d17d85b3105de_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,780 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412075b8449747dbc442da15a6b47fb525e9d_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412075b8449747dbc442da15a6b47fb525e9d_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,781 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076754b815e1fa429facb06329bb77a2cb_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076754b815e1fa429facb06329bb77a2cb_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,782 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077367c00d166c4e64a3c330098d14acf4_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077367c00d166c4e64a3c330098d14acf4_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,783 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077a429fa478524ef9b9c6639a9b68780f_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412077a429fa478524ef9b9c6639a9b68780f_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,784 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120788c98bc214b14dd38414ef8bed6f753f_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120788c98bc214b14dd38414ef8bed6f753f_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,785 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412078f9c821187dd4924afd1185cce06062b_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412078f9c821187dd4924afd1185cce06062b_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,786 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207970f31a47b32498a8e4ab94135c76a0a_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207970f31a47b32498a8e4ab94135c76a0a_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,787 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207abc4dc41877347af9386c76bedf2c8de_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207abc4dc41877347af9386c76bedf2c8de_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,788 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ad674937c07844dca9824bb4eecd4067_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ad674937c07844dca9824bb4eecd4067_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,789 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b34ee210695a4d55882ff579f0221728_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b34ee210695a4d55882ff579f0221728_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,790 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207cecc63062f2b439eb9e6beb67dbbdd36_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207cecc63062f2b439eb9e6beb67dbbdd36_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,791 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e2838460d9c8436d9833043a3b78b131_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e2838460d9c8436d9833043a3b78b131_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,792 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e73138f8c040493986226bdffe2fb62f_3d520264937e773a7192a548dec4f0b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e73138f8c040493986226bdffe2fb62f_3d520264937e773a7192a548dec4f0b4 2024-12-07T18:14:45,793 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T18:14:45,794 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:45,798 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T18:14:45,799 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T18:14:45,800 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:45,800 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T18:14:45,800 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733595285800"}]},"ts":"9223372036854775807"} 2024-12-07T18:14:45,802 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T18:14:45,802 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3d520264937e773a7192a548dec4f0b4, NAME => 'TestAcidGuarantees,,1733595252609.3d520264937e773a7192a548dec4f0b4.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T18:14:45,802 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T18:14:45,803 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733595285802"}]},"ts":"9223372036854775807"} 2024-12-07T18:14:45,804 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T18:14:45,804 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:45,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-12-07T18:14:45,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-07T18:14:45,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T18:14:45,858 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:45,871 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-17801953_22 at /127.0.0.1:51748 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/cluster_452d438f-2bce-cc52-05da-ff1816a8b998/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/cluster_452d438f-2bce-cc52-05da-ff1816a8b998/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=414 (was 304) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6641 (was 7170) 2024-12-07T18:14:45,880 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=414, ProcessCount=11, AvailableMemoryMB=6640 2024-12-07T18:14:45,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T18:14:45,881 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:14:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T18:14:45,883 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T18:14:45,883 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:45,883 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-12-07T18:14:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T18:14:45,884 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T18:14:45,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742077_1253 (size=1061) 2024-12-07T18:14:45,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T18:14:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T18:14:46,291 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68 2024-12-07T18:14:46,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742078_1254 (size=53) 2024-12-07T18:14:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T18:14:46,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:14:46,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0fc774415149ceeafbb0c8c46caa372c, disabling compactions & flushes 2024-12-07T18:14:46,697 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:46,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:46,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. after waiting 0 ms 2024-12-07T18:14:46,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:46,697 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:46,697 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0fc774415149ceeafbb0c8c46caa372c: Waiting for close lock at 1733595286697Disabling compacts and flushes for region at 1733595286697Disabling writes for close at 1733595286697Writing region close event to WAL at 1733595286697Closed at 1733595286697 2024-12-07T18:14:46,698 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T18:14:46,699 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733595286698"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733595286698"}]},"ts":"1733595286698"} 2024-12-07T18:14:46,700 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T18:14:46,701 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T18:14:46,701 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595286701"}]},"ts":"1733595286701"} 2024-12-07T18:14:46,702 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T18:14:46,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc774415149ceeafbb0c8c46caa372c, ASSIGN}] 2024-12-07T18:14:46,704 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc774415149ceeafbb0c8c46caa372c, ASSIGN 2024-12-07T18:14:46,704 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc774415149ceeafbb0c8c46caa372c, ASSIGN; state=OFFLINE, location=92acd7e2389b,33947,1733595217075; forceNewPlan=false, retain=false 2024-12-07T18:14:46,751 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-07T18:14:46,855 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=0fc774415149ceeafbb0c8c46caa372c, regionState=OPENING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:46,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc774415149ceeafbb0c8c46caa372c, ASSIGN because future has completed 2024-12-07T18:14:46,857 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:14:47,012 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:47,012 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:14:47,013 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,013 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:14:47,013 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,013 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,014 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,016 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:14:47,017 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fc774415149ceeafbb0c8c46caa372c columnFamilyName A 2024-12-07T18:14:47,017 DEBUG [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:47,017 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] regionserver.HStore(327): Store=0fc774415149ceeafbb0c8c46caa372c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:14:47,018 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T18:14:47,019 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:14:47,019 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fc774415149ceeafbb0c8c46caa372c columnFamilyName B 2024-12-07T18:14:47,019 DEBUG [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:47,020 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] regionserver.HStore(327): Store=0fc774415149ceeafbb0c8c46caa372c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:14:47,020 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,021 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:14:47,021 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fc774415149ceeafbb0c8c46caa372c columnFamilyName C 2024-12-07T18:14:47,021 DEBUG [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:14:47,022 INFO [StoreOpener-0fc774415149ceeafbb0c8c46caa372c-1 {}] regionserver.HStore(327): Store=0fc774415149ceeafbb0c8c46caa372c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:14:47,022 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:47,022 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,023 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,023 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,024 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,024 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,025 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:14:47,026 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,028 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:14:47,028 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 0fc774415149ceeafbb0c8c46caa372c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67083450, jitterRate=-3.7869811058044434E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:14:47,029 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:47,029 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 0fc774415149ceeafbb0c8c46caa372c: Running coprocessor pre-open hook at 1733595287013Writing region info on filesystem at 1733595287013Initializing all the Stores at 1733595287014 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595287014Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595287014Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595287014Cleaning up temporary data from old regions at 1733595287024 (+10 ms)Running coprocessor post-open hooks at 1733595287029 (+5 ms)Region opened successfully at 1733595287029 2024-12-07T18:14:47,030 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., pid=61, masterSystemTime=1733595287008 2024-12-07T18:14:47,032 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:47,032 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:47,033 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=0fc774415149ceeafbb0c8c46caa372c, regionState=OPEN, openSeqNum=2, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:47,034 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:14:47,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-07T18:14:47,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 in 178 msec 2024-12-07T18:14:47,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-07T18:14:47,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc774415149ceeafbb0c8c46caa372c, ASSIGN in 334 msec 2024-12-07T18:14:47,039 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T18:14:47,039 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595287039"}]},"ts":"1733595287039"} 2024-12-07T18:14:47,041 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T18:14:47,042 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T18:14:47,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-12-07T18:14:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-07T18:14:48,028 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:48,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a754db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,030 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@112c78a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,032 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d58190, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,033 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,036 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T18:14:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 92acd7e2389b,38531,1733595216342 2024-12-07T18:14:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@589a6082 2024-12-07T18:14:48,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T18:14:48,037 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T18:14:48,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50502613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,039 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d72e1f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,040 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56422, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,040 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70cbebb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,042 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31aa940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,043 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,043 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55cd306e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,044 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,044 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,044 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e2dea0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,045 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66249c9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,047 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77e7caf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,048 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89d73e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,049 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c813e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,051 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdfc5e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,051 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,052 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@210aba2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,053 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bced3cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,054 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,054 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,054 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,055 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7903af50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,056 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206dc57b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,057 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df924b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,058 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,059 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53fa97c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,060 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10731ac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,061 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,061 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56558, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,062 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ade5599, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:14:48,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:14:48,063 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:14:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:14:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:14:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@279bd8ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:14:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:14:48,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:14:48,064 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:14:48,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ac62cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:14:48,065 DEBUG [Thread-1197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,065 DEBUG [Thread-1186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,065 DEBUG [Thread-1182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,066 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:48,066 DEBUG [Thread-1191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,066 DEBUG [Thread-1195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,066 DEBUG [Thread-1178 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,067 DEBUG [Thread-1193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-07T18:14:48,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,068 DEBUG [Thread-1189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T18:14:48,068 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,072 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:14:48,072 DEBUG [Thread-1195 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:14:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:14:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:14:48,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:14:48,074 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:48,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:48,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:14:48,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:14:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:14:48,078 DEBUG [Thread-1180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,078 DEBUG [Thread-1184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:14:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,079 DEBUG [Thread-1184 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:14:48,080 DEBUG [Thread-1184 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:14:48,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:14:48,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:14:48,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:14:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:48,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:14:48,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:48,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:48,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:48,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:48,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:48,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:48,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/7c0626dcc9814e488e875a4151737188 is 50, key is test_row_1/A:col10/1733595288088/Put/seqid=0 2024-12-07T18:14:48,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742079_1255 (size=7315) 2024-12-07T18:14:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595348141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595288114, max timestamp=1733595288151Num uniques 859; 2024-12-07T18:14:48,169 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:48,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595288114, max timestamp=1733595288151Num uniques 859; 2024-12-07T18:14:48,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:48,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733595288114, max timestamp=1733595288151Num uniques 859; 2024-12-07T18:14:48,176 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-07T18:14:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T18:14:48,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733595288200, max timestamp=1733595288213Num uniques 1018; 2024-12-07T18:14:48,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-07T18:14:48,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733595288200, max timestamp=1733595288213Num uniques 1018; 2024-12-07T18:14:48,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-07T18:14:48,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733595288200, max timestamp=1733595288213Num uniques 1009; 2024-12-07T18:14:48,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-07T18:14:48,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T18:14:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:48,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595348245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595348245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595348248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595348249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595348280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595348359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595348359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595348359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595348359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T18:14:48,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:48,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T18:14:48,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595348488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/7c0626dcc9814e488e875a4151737188 2024-12-07T18:14:48,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T18:14:48,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:48,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/2c151adbffc842488296743c6f72d95c is 50, key is test_row_1/B:col10/1733595288088/Put/seqid=0 2024-12-07T18:14:48,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595348568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595348568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595348569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595348569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742080_1256 (size=7315) 2024-12-07T18:14:48,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T18:14:48,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:48,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T18:14:48,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595348799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T18:14:48,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:48,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595348878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595348878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595348879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595348879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:48,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:48,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T18:14:48,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:48,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:48,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:48,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/2c151adbffc842488296743c6f72d95c 2024-12-07T18:14:49,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/ea76d6de0569425189a075ccaf825396 is 50, key is test_row_1/C:col10/1733595288088/Put/seqid=0 2024-12-07T18:14:49,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742081_1257 (size=7315) 2024-12-07T18:14:49,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T18:14:49,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:49,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:49,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:49,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:49,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:49,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T18:14:49,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T18:14:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:49,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:49,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:49,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595349308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:49,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:49,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:49,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:49,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:49,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595349388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595349389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:49,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:49,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595349389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595349389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:49,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:49,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/ea76d6de0569425189a075ccaf825396 2024-12-07T18:14:49,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/7c0626dcc9814e488e875a4151737188 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/7c0626dcc9814e488e875a4151737188 2024-12-07T18:14:49,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/7c0626dcc9814e488e875a4151737188, entries=50, sequenceid=12, filesize=7.1 K 2024-12-07T18:14:49,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/2c151adbffc842488296743c6f72d95c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c151adbffc842488296743c6f72d95c 2024-12-07T18:14:49,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c151adbffc842488296743c6f72d95c, entries=50, sequenceid=12, filesize=7.1 K 2024-12-07T18:14:49,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/ea76d6de0569425189a075ccaf825396 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea76d6de0569425189a075ccaf825396 2024-12-07T18:14:49,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T18:14:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:49,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:49,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:49,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea76d6de0569425189a075ccaf825396, entries=50, sequenceid=12, filesize=7.1 K 2024-12-07T18:14:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:49,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.44 KB/127422 for 0fc774415149ceeafbb0c8c46caa372c in 1370ms, sequenceid=12, compaction requested=false 2024-12-07T18:14:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:49,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-07T18:14:49,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:49,608 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=124.44 KB heapSize=352.63 KB 2024-12-07T18:14:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:49,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:49,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/a141e9a799d843fcad10c64d495bdff2 is 50, key is test_row_0/A:col10/1733595288242/Put/seqid=0 2024-12-07T18:14:49,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742082_1258 (size=12001) 2024-12-07T18:14:49,620 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/a141e9a799d843fcad10c64d495bdff2 2024-12-07T18:14:49,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/1600a1bafaaa41bcbfc2a9b2bf73fada is 50, key is test_row_0/B:col10/1733595288242/Put/seqid=0 2024-12-07T18:14:49,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742083_1259 (size=12001) 2024-12-07T18:14:50,035 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/1600a1bafaaa41bcbfc2a9b2bf73fada 2024-12-07T18:14:50,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/370d710984b34d439295d27f8f492102 is 50, key is test_row_0/C:col10/1733595288242/Put/seqid=0 2024-12-07T18:14:50,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742084_1260 (size=12001) 2024-12-07T18:14:50,054 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.46 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/370d710984b34d439295d27f8f492102 2024-12-07T18:14:50,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/a141e9a799d843fcad10c64d495bdff2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/a141e9a799d843fcad10c64d495bdff2 2024-12-07T18:14:50,064 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/a141e9a799d843fcad10c64d495bdff2, entries=150, sequenceid=64, filesize=11.7 K 2024-12-07T18:14:50,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/1600a1bafaaa41bcbfc2a9b2bf73fada as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/1600a1bafaaa41bcbfc2a9b2bf73fada 2024-12-07T18:14:50,071 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/1600a1bafaaa41bcbfc2a9b2bf73fada, entries=150, sequenceid=64, filesize=11.7 K 2024-12-07T18:14:50,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/370d710984b34d439295d27f8f492102 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/370d710984b34d439295d27f8f492102 2024-12-07T18:14:50,076 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/370d710984b34d439295d27f8f492102, entries=150, sequenceid=64, filesize=11.7 K 2024-12-07T18:14:50,076 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~124.44 KB/127422, heapSize ~352.58 KB/361040, currentSize=0 B/0 for 0fc774415149ceeafbb0c8c46caa372c in 468ms, sequenceid=64, compaction requested=false 2024-12-07T18:14:50,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:50,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-07T18:14:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-07T18:14:50,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-07T18:14:50,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0040 sec 2024-12-07T18:14:50,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.0150 sec 2024-12-07T18:14:50,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-07T18:14:50,218 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:50,219 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:50,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-07T18:14:50,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T18:14:50,221 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:50,222 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:50,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T18:14:50,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:50,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:14:50,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:50,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:50,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:50,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:50,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:50,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:50,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/49970c60f8204fc2849ec71619ffe554 is 50, key is test_row_0/A:col10/1733595290333/Put/seqid=0 2024-12-07T18:14:50,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742085_1261 (size=14341) 2024-12-07T18:14:50,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T18:14:50,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:50,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595290401, max timestamp=1733595290413Num uniques 859; 2024-12-07T18:14:50,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595290401, max timestamp=1733595290413Num uniques 859; 2024-12-07T18:14:50,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:50,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:50,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595290401, max timestamp=1733595290413Num uniques 859; 2024-12-07T18:14:50,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:50,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595350433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595350434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595350435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595350436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595350440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T18:14:50,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:50,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T18:14:50,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595350548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595350548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595350549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595350549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595350549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T18:14:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/49970c60f8204fc2849ec71619ffe554 2024-12-07T18:14:50,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/c22433f9c9084cf38fda01d1883f9d4c is 50, key is test_row_0/B:col10/1733595290333/Put/seqid=0 2024-12-07T18:14:50,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595350758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595350759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595350759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595350759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595350759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:50,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:50,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742086_1262 (size=12001) 2024-12-07T18:14:50,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T18:14:50,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:50,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T18:14:50,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T18:14:50,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:50,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:50,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:51,056 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T18:14:51,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595351068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595351069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595351069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595351069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595351078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T18:14:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:51,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:51,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:51,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/c22433f9c9084cf38fda01d1883f9d4c 2024-12-07T18:14:51,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/c2ab959650e44be0a05d36bb2b3a9608 is 50, key is test_row_0/C:col10/1733595290333/Put/seqid=0 2024-12-07T18:14:51,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742087_1263 (size=12001) 2024-12-07T18:14:51,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/c2ab959650e44be0a05d36bb2b3a9608 2024-12-07T18:14:51,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/49970c60f8204fc2849ec71619ffe554 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/49970c60f8204fc2849ec71619ffe554 2024-12-07T18:14:51,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/49970c60f8204fc2849ec71619ffe554, entries=200, sequenceid=74, filesize=14.0 K 2024-12-07T18:14:51,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/c22433f9c9084cf38fda01d1883f9d4c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/c22433f9c9084cf38fda01d1883f9d4c 2024-12-07T18:14:51,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/c22433f9c9084cf38fda01d1883f9d4c, entries=150, sequenceid=74, filesize=11.7 K 2024-12-07T18:14:51,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/c2ab959650e44be0a05d36bb2b3a9608 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c2ab959650e44be0a05d36bb2b3a9608 2024-12-07T18:14:51,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c2ab959650e44be0a05d36bb2b3a9608, entries=150, sequenceid=74, filesize=11.7 K 2024-12-07T18:14:51,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0fc774415149ceeafbb0c8c46caa372c in 862ms, sequenceid=74, compaction requested=true 2024-12-07T18:14:51,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:51,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:51,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:51,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:51,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:51,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:51,205 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:51,205 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:51,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:51,206 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:51,206 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:51,206 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/B is initiating minor compaction (all files) 2024-12-07T18:14:51,206 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/A is initiating minor compaction (all files) 2024-12-07T18:14:51,206 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/B in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:51,206 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/A in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:51,206 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c151adbffc842488296743c6f72d95c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/1600a1bafaaa41bcbfc2a9b2bf73fada, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/c22433f9c9084cf38fda01d1883f9d4c] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=30.6 K 2024-12-07T18:14:51,206 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/7c0626dcc9814e488e875a4151737188, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/a141e9a799d843fcad10c64d495bdff2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/49970c60f8204fc2849ec71619ffe554] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=32.9 K 2024-12-07T18:14:51,207 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c0626dcc9814e488e875a4151737188, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733595288088 2024-12-07T18:14:51,207 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c151adbffc842488296743c6f72d95c, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733595288088 2024-12-07T18:14:51,209 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting a141e9a799d843fcad10c64d495bdff2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595288242 2024-12-07T18:14:51,209 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 1600a1bafaaa41bcbfc2a9b2bf73fada, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595288242 2024-12-07T18:14:51,209 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49970c60f8204fc2849ec71619ffe554, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733595290333 2024-12-07T18:14:51,209 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting c22433f9c9084cf38fda01d1883f9d4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733595290333 2024-12-07T18:14:51,221 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:51,221 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/74b9eafaddb9424c8281704487d6e462 is 50, key is test_row_0/A:col10/1733595290333/Put/seqid=0 2024-12-07T18:14:51,222 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:51,223 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/dd356eab006b49028cfc0b60cc68e4fd is 50, key is test_row_0/B:col10/1733595290333/Put/seqid=0 2024-12-07T18:14:51,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742088_1264 (size=12104) 2024-12-07T18:14:51,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742089_1265 (size=12104) 2024-12-07T18:14:51,240 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/74b9eafaddb9424c8281704487d6e462 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/74b9eafaddb9424c8281704487d6e462 2024-12-07T18:14:51,246 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/A of 0fc774415149ceeafbb0c8c46caa372c into 74b9eafaddb9424c8281704487d6e462(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:51,246 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:51,246 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/A, priority=13, startTime=1733595291205; duration=0sec 2024-12-07T18:14:51,247 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:51,247 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:A 2024-12-07T18:14:51,247 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:51,249 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:51,249 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/C is initiating minor compaction (all files) 2024-12-07T18:14:51,249 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/C in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:51,249 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea76d6de0569425189a075ccaf825396, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/370d710984b34d439295d27f8f492102, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c2ab959650e44be0a05d36bb2b3a9608] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=30.6 K 2024-12-07T18:14:51,249 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea76d6de0569425189a075ccaf825396, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733595288088 2024-12-07T18:14:51,250 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 370d710984b34d439295d27f8f492102, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595288242 2024-12-07T18:14:51,251 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ab959650e44be0a05d36bb2b3a9608, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733595290333 2024-12-07T18:14:51,259 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#C#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:51,260 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/16542af5974643b5b57f8e0bdd4b1738 is 50, key is test_row_0/C:col10/1733595290333/Put/seqid=0 2024-12-07T18:14:51,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742090_1266 (size=12104) 2024-12-07T18:14:51,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-07T18:14:51,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:51,295 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-07T18:14:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:51,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:51,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/43bdf4a41d7342c7a80750ec52bea0ee is 50, key is test_row_0/A:col10/1733595290432/Put/seqid=0 2024-12-07T18:14:51,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742091_1267 (size=12001) 2024-12-07T18:14:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T18:14:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:51,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:51,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595351593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595351594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595351594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595351595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595351595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,641 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/dd356eab006b49028cfc0b60cc68e4fd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/dd356eab006b49028cfc0b60cc68e4fd 2024-12-07T18:14:51,646 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/B of 0fc774415149ceeafbb0c8c46caa372c into dd356eab006b49028cfc0b60cc68e4fd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:51,646 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:51,646 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/B, priority=13, startTime=1733595291205; duration=0sec 2024-12-07T18:14:51,647 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:51,647 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:B 2024-12-07T18:14:51,670 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/16542af5974643b5b57f8e0bdd4b1738 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/16542af5974643b5b57f8e0bdd4b1738 2024-12-07T18:14:51,675 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/C of 0fc774415149ceeafbb0c8c46caa372c into 16542af5974643b5b57f8e0bdd4b1738(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:51,675 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:51,675 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/C, priority=13, startTime=1733595291205; duration=0sec 2024-12-07T18:14:51,675 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:51,675 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:C 2024-12-07T18:14:51,705 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/43bdf4a41d7342c7a80750ec52bea0ee 2024-12-07T18:14:51,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595351708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595351708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595351708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595351708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595351708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,712 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/fc3242fa12344ca8a35b9dcbb34acde2 is 50, key is test_row_0/B:col10/1733595290432/Put/seqid=0 2024-12-07T18:14:51,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742092_1268 (size=12001) 2024-12-07T18:14:51,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595351918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595351918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595351918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595351918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595351919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:51,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,120 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/fc3242fa12344ca8a35b9dcbb34acde2 2024-12-07T18:14:52,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/8e228c2d411144488e61f88a69ae9653 is 50, key is test_row_0/C:col10/1733595290432/Put/seqid=0 2024-12-07T18:14:52,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742093_1269 (size=12001) 2024-12-07T18:14:52,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595352228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595352228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595352228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595352228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595352228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T18:14:52,540 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/8e228c2d411144488e61f88a69ae9653 2024-12-07T18:14:52,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/43bdf4a41d7342c7a80750ec52bea0ee as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/43bdf4a41d7342c7a80750ec52bea0ee 2024-12-07T18:14:52,549 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/43bdf4a41d7342c7a80750ec52bea0ee, entries=150, sequenceid=112, filesize=11.7 K 2024-12-07T18:14:52,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/fc3242fa12344ca8a35b9dcbb34acde2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fc3242fa12344ca8a35b9dcbb34acde2 2024-12-07T18:14:52,553 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fc3242fa12344ca8a35b9dcbb34acde2, entries=150, sequenceid=112, filesize=11.7 K 2024-12-07T18:14:52,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/8e228c2d411144488e61f88a69ae9653 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/8e228c2d411144488e61f88a69ae9653 2024-12-07T18:14:52,559 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/8e228c2d411144488e61f88a69ae9653, entries=150, sequenceid=112, filesize=11.7 K 2024-12-07T18:14:52,560 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=60.38 KB/61830 for 0fc774415149ceeafbb0c8c46caa372c in 1265ms, sequenceid=112, compaction requested=false 2024-12-07T18:14:52,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:52,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:52,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-07T18:14:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-07T18:14:52,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-07T18:14:52,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3390 sec 2024-12-07T18:14:52,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.3440 sec 2024-12-07T18:14:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:52,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T18:14:52,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:52,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:52,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:52,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:52,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:52,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:52,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/d56e4a1348dc4ad7aa00ffca91c2bc69 is 50, key is test_row_0/A:col10/1733595292739/Put/seqid=0 2024-12-07T18:14:52,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742094_1270 (size=16681) 2024-12-07T18:14:52,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595352795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595352797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595352799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595352801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595352803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,812 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595352909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595352918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595352918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595352918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:52,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595352918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:52,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595353118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595353128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595353128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595353128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595353129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,133 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/d56e4a1348dc4ad7aa00ffca91c2bc69 2024-12-07T18:14:53,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/a913add0d0db4918997cc5a29a69b84a is 50, key is test_row_0/B:col10/1733595292739/Put/seqid=0 2024-12-07T18:14:53,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742095_1271 (size=12001) 2024-12-07T18:14:53,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/a913add0d0db4918997cc5a29a69b84a 2024-12-07T18:14:53,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/58cf3ad9b52f48e6ae252d63e6997f51 is 50, key is test_row_0/C:col10/1733595292739/Put/seqid=0 2024-12-07T18:14:53,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742096_1272 (size=12001) 2024-12-07T18:14:53,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/58cf3ad9b52f48e6ae252d63e6997f51 2024-12-07T18:14:53,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/d56e4a1348dc4ad7aa00ffca91c2bc69 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d56e4a1348dc4ad7aa00ffca91c2bc69 2024-12-07T18:14:53,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d56e4a1348dc4ad7aa00ffca91c2bc69, entries=250, sequenceid=128, filesize=16.3 K 2024-12-07T18:14:53,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/a913add0d0db4918997cc5a29a69b84a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a913add0d0db4918997cc5a29a69b84a 2024-12-07T18:14:53,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a913add0d0db4918997cc5a29a69b84a, entries=150, sequenceid=128, filesize=11.7 K 2024-12-07T18:14:53,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/58cf3ad9b52f48e6ae252d63e6997f51 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/58cf3ad9b52f48e6ae252d63e6997f51 2024-12-07T18:14:53,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/58cf3ad9b52f48e6ae252d63e6997f51, entries=150, sequenceid=128, filesize=11.7 K 2024-12-07T18:14:53,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 0fc774415149ceeafbb0c8c46caa372c in 468ms, sequenceid=128, compaction requested=true 2024-12-07T18:14:53,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:53,210 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:53,210 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:53,211 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:53,211 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:53,211 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/A is initiating minor compaction (all files) 2024-12-07T18:14:53,211 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/A in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:53,211 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/B is initiating minor compaction (all files) 2024-12-07T18:14:53,212 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/74b9eafaddb9424c8281704487d6e462, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/43bdf4a41d7342c7a80750ec52bea0ee, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d56e4a1348dc4ad7aa00ffca91c2bc69] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=39.8 K 2024-12-07T18:14:53,212 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/B in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:53,212 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/dd356eab006b49028cfc0b60cc68e4fd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fc3242fa12344ca8a35b9dcbb34acde2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a913add0d0db4918997cc5a29a69b84a] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=35.3 K 2024-12-07T18:14:53,212 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b9eafaddb9424c8281704487d6e462, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733595290333 2024-12-07T18:14:53,212 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting dd356eab006b49028cfc0b60cc68e4fd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733595290333 2024-12-07T18:14:53,212 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43bdf4a41d7342c7a80750ec52bea0ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733595290432 2024-12-07T18:14:53,213 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting fc3242fa12344ca8a35b9dcbb34acde2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733595290432 2024-12-07T18:14:53,213 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting d56e4a1348dc4ad7aa00ffca91c2bc69, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733595291592 2024-12-07T18:14:53,213 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting a913add0d0db4918997cc5a29a69b84a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733595291594 2024-12-07T18:14:53,225 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:53,226 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/377e859ba3f648dd8accc7d4db7afdf6 is 50, key is test_row_0/B:col10/1733595292739/Put/seqid=0 2024-12-07T18:14:53,231 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:53,231 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/052bd429fe0f4ad3933adab6da5f5d63 is 50, key is test_row_0/A:col10/1733595292739/Put/seqid=0 2024-12-07T18:14:53,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742097_1273 (size=12207) 2024-12-07T18:14:53,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742098_1274 (size=12207) 2024-12-07T18:14:53,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:53,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T18:14:53,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:53,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:53,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:53,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:53,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:53,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:53,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595353451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595353450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/e5a63458efaf499387b302b2bb85ec88 is 50, key is test_row_0/A:col10/1733595292799/Put/seqid=0 2024-12-07T18:14:53,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742099_1275 (size=12151) 2024-12-07T18:14:53,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595353458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595353458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595353458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/e5a63458efaf499387b302b2bb85ec88 2024-12-07T18:14:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/2c9e7a6372b1413980b87ff30d03d0fd is 50, key is test_row_0/B:col10/1733595292799/Put/seqid=0 2024-12-07T18:14:53,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742100_1276 (size=12151) 2024-12-07T18:14:53,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595353568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595353568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595353578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595353578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595353578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,646 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/377e859ba3f648dd8accc7d4db7afdf6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/377e859ba3f648dd8accc7d4db7afdf6 2024-12-07T18:14:53,648 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/052bd429fe0f4ad3933adab6da5f5d63 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/052bd429fe0f4ad3933adab6da5f5d63 2024-12-07T18:14:53,652 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/B of 0fc774415149ceeafbb0c8c46caa372c into 377e859ba3f648dd8accc7d4db7afdf6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:53,652 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:53,652 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/B, priority=13, startTime=1733595293210; duration=0sec 2024-12-07T18:14:53,652 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:53,652 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:B 2024-12-07T18:14:53,652 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:53,656 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:53,656 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/C is initiating minor compaction (all files) 2024-12-07T18:14:53,656 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/C in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:53,656 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/16542af5974643b5b57f8e0bdd4b1738, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/8e228c2d411144488e61f88a69ae9653, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/58cf3ad9b52f48e6ae252d63e6997f51] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=35.3 K 2024-12-07T18:14:53,656 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 16542af5974643b5b57f8e0bdd4b1738, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1733595290333 2024-12-07T18:14:53,657 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/A of 0fc774415149ceeafbb0c8c46caa372c into 052bd429fe0f4ad3933adab6da5f5d63(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:53,657 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:53,657 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e228c2d411144488e61f88a69ae9653, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733595290432 2024-12-07T18:14:53,657 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/A, priority=13, startTime=1733595293210; duration=0sec 2024-12-07T18:14:53,657 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:53,657 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:A 2024-12-07T18:14:53,657 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 58cf3ad9b52f48e6ae252d63e6997f51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733595291594 2024-12-07T18:14:53,664 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#C#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:53,665 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/d0e2dedb33c24c1eaac55f1cdff8bbf6 is 50, key is test_row_0/C:col10/1733595292739/Put/seqid=0 2024-12-07T18:14:53,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742101_1277 (size=12207) 2024-12-07T18:14:53,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595353778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595353778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595353789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595353789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595353789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:53,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/2c9e7a6372b1413980b87ff30d03d0fd 2024-12-07T18:14:53,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/c5ee8b6298464fbf8135ece5aa5e9deb is 50, key is test_row_0/C:col10/1733595292799/Put/seqid=0 2024-12-07T18:14:53,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742102_1278 (size=12151) 2024-12-07T18:14:54,074 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/d0e2dedb33c24c1eaac55f1cdff8bbf6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/d0e2dedb33c24c1eaac55f1cdff8bbf6 2024-12-07T18:14:54,078 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/C of 0fc774415149ceeafbb0c8c46caa372c into d0e2dedb33c24c1eaac55f1cdff8bbf6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:54,078 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:54,079 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/C, priority=13, startTime=1733595293210; duration=0sec 2024-12-07T18:14:54,079 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:54,079 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:C 2024-12-07T18:14:54,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595354088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595354088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595354098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595354098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595354099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/c5ee8b6298464fbf8135ece5aa5e9deb 2024-12-07T18:14:54,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/e5a63458efaf499387b302b2bb85ec88 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/e5a63458efaf499387b302b2bb85ec88 2024-12-07T18:14:54,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/e5a63458efaf499387b302b2bb85ec88, entries=150, sequenceid=148, filesize=11.9 K 2024-12-07T18:14:54,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/2c9e7a6372b1413980b87ff30d03d0fd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c9e7a6372b1413980b87ff30d03d0fd 2024-12-07T18:14:54,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c9e7a6372b1413980b87ff30d03d0fd, entries=150, sequenceid=148, filesize=11.9 K 2024-12-07T18:14:54,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/c5ee8b6298464fbf8135ece5aa5e9deb as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c5ee8b6298464fbf8135ece5aa5e9deb 2024-12-07T18:14:54,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c5ee8b6298464fbf8135ece5aa5e9deb, entries=150, sequenceid=148, filesize=11.9 K 2024-12-07T18:14:54,322 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 0fc774415149ceeafbb0c8c46caa372c in 889ms, sequenceid=148, compaction requested=false 2024-12-07T18:14:54,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-07T18:14:54,388 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:54,388 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-07T18:14:54,390 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:54,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T18:14:54,391 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:54,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T18:14:54,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-07T18:14:54,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:54,545 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T18:14:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:54,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/1111b569073f48cbadd717e0ff5973d1 is 50, key is test_row_0/A:col10/1733595293455/Put/seqid=0 2024-12-07T18:14:54,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742103_1279 (size=12151) 2024-12-07T18:14:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:54,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:54,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595354619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595354620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595354621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595354623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595354624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T18:14:54,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595354728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595354738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595354739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595354739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595354739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595354938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595354948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595354949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595354949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595354949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:54,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:54,955 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/1111b569073f48cbadd717e0ff5973d1 2024-12-07T18:14:54,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/939555ff6c344ba1a6475e791c74bc39 is 50, key is test_row_0/B:col10/1733595293455/Put/seqid=0 2024-12-07T18:14:54,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742104_1280 (size=12151) 2024-12-07T18:14:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T18:14:55,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595355248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595355258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595355259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595355259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595355259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,368 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/939555ff6c344ba1a6475e791c74bc39 2024-12-07T18:14:55,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/7f03a1a7e2634e228299976c89448743 is 50, key is test_row_0/C:col10/1733595293455/Put/seqid=0 2024-12-07T18:14:55,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742105_1281 (size=12151) 2024-12-07T18:14:55,380 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/7f03a1a7e2634e228299976c89448743 2024-12-07T18:14:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/1111b569073f48cbadd717e0ff5973d1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/1111b569073f48cbadd717e0ff5973d1 2024-12-07T18:14:55,392 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/1111b569073f48cbadd717e0ff5973d1, entries=150, sequenceid=164, filesize=11.9 K 2024-12-07T18:14:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/939555ff6c344ba1a6475e791c74bc39 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/939555ff6c344ba1a6475e791c74bc39 2024-12-07T18:14:55,397 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/939555ff6c344ba1a6475e791c74bc39, entries=150, sequenceid=164, filesize=11.9 K 2024-12-07T18:14:55,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/7f03a1a7e2634e228299976c89448743 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/7f03a1a7e2634e228299976c89448743 2024-12-07T18:14:55,402 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/7f03a1a7e2634e228299976c89448743, entries=150, sequenceid=164, filesize=11.9 K 2024-12-07T18:14:55,402 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0fc774415149ceeafbb0c8c46caa372c in 858ms, sequenceid=164, compaction requested=true 2024-12-07T18:14:55,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:55,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:55,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-07T18:14:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-07T18:14:55,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-07T18:14:55,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0130 sec 2024-12-07T18:14:55,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.0170 sec 2024-12-07T18:14:55,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-07T18:14:55,528 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:55,529 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-07T18:14:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-07T18:14:55,532 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:55,533 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:55,533 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-07T18:14:55,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-07T18:14:55,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:55,686 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T18:14:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:55,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:55,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/722a4fdcccd34be9b40d93909b160bdc is 50, key is test_row_0/A:col10/1733595294617/Put/seqid=0 2024-12-07T18:14:55,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742106_1282 (size=12151) 2024-12-07T18:14:55,708 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/722a4fdcccd34be9b40d93909b160bdc 2024-12-07T18:14:55,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/3fc839d118414490b0963e2c8b19821d is 50, key is test_row_0/B:col10/1733595294617/Put/seqid=0 2024-12-07T18:14:55,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742107_1283 (size=12151) 2024-12-07T18:14:55,738 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/3fc839d118414490b0963e2c8b19821d 2024-12-07T18:14:55,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/287a70043c694b54a2807d8f6ac5617b is 50, key is test_row_0/C:col10/1733595294617/Put/seqid=0 2024-12-07T18:14:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:55,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:55,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742108_1284 (size=12151) 2024-12-07T18:14:55,777 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/287a70043c694b54a2807d8f6ac5617b 2024-12-07T18:14:55,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/722a4fdcccd34be9b40d93909b160bdc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/722a4fdcccd34be9b40d93909b160bdc 2024-12-07T18:14:55,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595355778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595355778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,786 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/722a4fdcccd34be9b40d93909b160bdc, entries=150, sequenceid=183, filesize=11.9 K 2024-12-07T18:14:55,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/3fc839d118414490b0963e2c8b19821d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3fc839d118414490b0963e2c8b19821d 2024-12-07T18:14:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595355783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595355783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595355784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,792 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3fc839d118414490b0963e2c8b19821d, entries=150, sequenceid=183, filesize=11.9 K 2024-12-07T18:14:55,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/287a70043c694b54a2807d8f6ac5617b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/287a70043c694b54a2807d8f6ac5617b 2024-12-07T18:14:55,796 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/287a70043c694b54a2807d8f6ac5617b, entries=150, sequenceid=183, filesize=11.9 K 2024-12-07T18:14:55,797 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0fc774415149ceeafbb0c8c46caa372c in 111ms, sequenceid=183, compaction requested=true 2024-12-07T18:14:55,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:55,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:55,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-07T18:14:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-07T18:14:55,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-07T18:14:55,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 266 msec 2024-12-07T18:14:55,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 272 msec 2024-12-07T18:14:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-07T18:14:55,859 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:55,859 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-07T18:14:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-07T18:14:55,861 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:55,862 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:55,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:55,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T18:14:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:55,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:55,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/449d1414dfa540d08d327244ddfed2fc is 50, key is test_row_0/A:col10/1733595295772/Put/seqid=0 2024-12-07T18:14:55,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742109_1285 (size=14541) 2024-12-07T18:14:55,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595355920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595355939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595355939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595355939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595355940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:55,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:55,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-07T18:14:56,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-07T18:14:56,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:56,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595356049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595356049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595356049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595356049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595356058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-07T18:14:56,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:56,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-07T18:14:56,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595356258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595356258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595356258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595356258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595356268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/449d1414dfa540d08d327244ddfed2fc 2024-12-07T18:14:56,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-07T18:14:56,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:56,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/fd0a576953f846fa81d115e327a4be8e is 50, key is test_row_0/B:col10/1733595295772/Put/seqid=0 2024-12-07T18:14:56,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742110_1286 (size=12151) 2024-12-07T18:14:56,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-07T18:14:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-07T18:14:56,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595356569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595356569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595356569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595356578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595356578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:56,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-07T18:14:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/fd0a576953f846fa81d115e327a4be8e 2024-12-07T18:14:56,751 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-07T18:14:56,751 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-07T18:14:56,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/bf448a6383804fa9853db0f417bf8e1f is 50, key is test_row_0/C:col10/1733595295772/Put/seqid=0 2024-12-07T18:14:56,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742111_1287 (size=12151) 2024-12-07T18:14:56,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-07T18:14:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-07T18:14:56,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:56,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:56,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-07T18:14:57,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595357078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:57,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595357078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:57,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:57,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595357088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595357089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:57,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-07T18:14:57,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:57,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:57,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:57,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:57,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:57,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595357099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:57,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:57,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:57,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:57,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/bf448a6383804fa9853db0f417bf8e1f 2024-12-07T18:14:57,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/449d1414dfa540d08d327244ddfed2fc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/449d1414dfa540d08d327244ddfed2fc 2024-12-07T18:14:57,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/449d1414dfa540d08d327244ddfed2fc, entries=200, sequenceid=197, filesize=14.2 K 2024-12-07T18:14:57,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/fd0a576953f846fa81d115e327a4be8e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fd0a576953f846fa81d115e327a4be8e 2024-12-07T18:14:57,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fd0a576953f846fa81d115e327a4be8e, entries=150, sequenceid=197, filesize=11.9 K 2024-12-07T18:14:57,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/bf448a6383804fa9853db0f417bf8e1f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/bf448a6383804fa9853db0f417bf8e1f 2024-12-07T18:14:57,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/bf448a6383804fa9853db0f417bf8e1f, entries=150, sequenceid=197, filesize=11.9 K 2024-12-07T18:14:57,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0fc774415149ceeafbb0c8c46caa372c in 1303ms, sequenceid=197, compaction requested=true 2024-12-07T18:14:57,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:57,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:57,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:57,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:57,192 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-07T18:14:57,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:57,192 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-07T18:14:57,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:57,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:57,200 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63201 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-07T18:14:57,200 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60811 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-07T18:14:57,200 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/B is initiating minor compaction (all files) 2024-12-07T18:14:57,200 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/A is initiating minor compaction (all files) 2024-12-07T18:14:57,200 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/A in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:57,200 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/B in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:57,201 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/377e859ba3f648dd8accc7d4db7afdf6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c9e7a6372b1413980b87ff30d03d0fd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/939555ff6c344ba1a6475e791c74bc39, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3fc839d118414490b0963e2c8b19821d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fd0a576953f846fa81d115e327a4be8e] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=59.4 K 2024-12-07T18:14:57,201 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/052bd429fe0f4ad3933adab6da5f5d63, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/e5a63458efaf499387b302b2bb85ec88, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/1111b569073f48cbadd717e0ff5973d1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/722a4fdcccd34be9b40d93909b160bdc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/449d1414dfa540d08d327244ddfed2fc] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=61.7 K 2024-12-07T18:14:57,201 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 052bd429fe0f4ad3933adab6da5f5d63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733595291594 2024-12-07T18:14:57,201 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 377e859ba3f648dd8accc7d4db7afdf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733595291594 2024-12-07T18:14:57,202 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5a63458efaf499387b302b2bb85ec88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733595292799 2024-12-07T18:14:57,202 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9e7a6372b1413980b87ff30d03d0fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733595292799 2024-12-07T18:14:57,202 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1111b569073f48cbadd717e0ff5973d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733595293447 2024-12-07T18:14:57,202 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 939555ff6c344ba1a6475e791c74bc39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733595293447 2024-12-07T18:14:57,202 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 722a4fdcccd34be9b40d93909b160bdc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733595294617 2024-12-07T18:14:57,202 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fc839d118414490b0963e2c8b19821d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733595294617 2024-12-07T18:14:57,203 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 449d1414dfa540d08d327244ddfed2fc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733595295772 2024-12-07T18:14:57,203 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0a576953f846fa81d115e327a4be8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733595295772 2024-12-07T18:14:57,220 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#A#compaction#240 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:57,221 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/eba085d67d75440e861df4a4a2e4428a is 50, key is test_row_0/A:col10/1733595295772/Put/seqid=0 2024-12-07T18:14:57,230 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#B#compaction#241 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:57,230 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/6b5dd338a84847cb894fab57ff907fb2 is 50, key is test_row_0/B:col10/1733595295772/Put/seqid=0 2024-12-07T18:14:57,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-07T18:14:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:57,248 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T18:14:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:57,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:57,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742112_1288 (size=12527) 2024-12-07T18:14:57,277 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/eba085d67d75440e861df4a4a2e4428a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/eba085d67d75440e861df4a4a2e4428a 2024-12-07T18:14:57,283 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/A of 0fc774415149ceeafbb0c8c46caa372c into eba085d67d75440e861df4a4a2e4428a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:57,284 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:57,284 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/A, priority=11, startTime=1733595297192; duration=0sec 2024-12-07T18:14:57,284 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:57,284 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:A 2024-12-07T18:14:57,284 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-07T18:14:57,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/d7ee68fd188649599e0147e2d5169c2e is 50, key is test_row_0/A:col10/1733595295905/Put/seqid=0 2024-12-07T18:14:57,289 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60811 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-07T18:14:57,289 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/C is initiating minor compaction (all files) 2024-12-07T18:14:57,289 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/C in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:57,289 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/d0e2dedb33c24c1eaac55f1cdff8bbf6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c5ee8b6298464fbf8135ece5aa5e9deb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/7f03a1a7e2634e228299976c89448743, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/287a70043c694b54a2807d8f6ac5617b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/bf448a6383804fa9853db0f417bf8e1f] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=59.4 K 2024-12-07T18:14:57,290 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0e2dedb33c24c1eaac55f1cdff8bbf6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1733595291594 2024-12-07T18:14:57,290 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5ee8b6298464fbf8135ece5aa5e9deb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733595292799 2024-12-07T18:14:57,291 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f03a1a7e2634e228299976c89448743, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1733595293447 2024-12-07T18:14:57,291 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 287a70043c694b54a2807d8f6ac5617b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1733595294617 2024-12-07T18:14:57,291 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf448a6383804fa9853db0f417bf8e1f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733595295772 2024-12-07T18:14:57,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742113_1289 (size=12527) 2024-12-07T18:14:57,327 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:57,328 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/e7c638032f0f40c1b6d49d653ea64f50 is 50, key is test_row_0/C:col10/1733595295772/Put/seqid=0 2024-12-07T18:14:57,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742114_1290 (size=12151) 2024-12-07T18:14:57,360 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/d7ee68fd188649599e0147e2d5169c2e 2024-12-07T18:14:57,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742115_1291 (size=12527) 2024-12-07T18:14:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/228d2e0f1b8d43a9b0a73c908444a270 is 50, key is test_row_0/B:col10/1733595295905/Put/seqid=0 2024-12-07T18:14:57,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742116_1292 (size=12151) 2024-12-07T18:14:57,438 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/228d2e0f1b8d43a9b0a73c908444a270 2024-12-07T18:14:57,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/04b8cb2dc1824c4ebccdedd9c0156fbd is 50, key is test_row_0/C:col10/1733595295905/Put/seqid=0 2024-12-07T18:14:57,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742117_1293 (size=12151) 2024-12-07T18:14:57,468 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/04b8cb2dc1824c4ebccdedd9c0156fbd 2024-12-07T18:14:57,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/d7ee68fd188649599e0147e2d5169c2e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d7ee68fd188649599e0147e2d5169c2e 2024-12-07T18:14:57,484 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d7ee68fd188649599e0147e2d5169c2e, entries=150, sequenceid=215, filesize=11.9 K 2024-12-07T18:14:57,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/228d2e0f1b8d43a9b0a73c908444a270 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/228d2e0f1b8d43a9b0a73c908444a270 2024-12-07T18:14:57,491 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/228d2e0f1b8d43a9b0a73c908444a270, entries=150, sequenceid=215, filesize=11.9 K 2024-12-07T18:14:57,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/04b8cb2dc1824c4ebccdedd9c0156fbd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/04b8cb2dc1824c4ebccdedd9c0156fbd 2024-12-07T18:14:57,499 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/04b8cb2dc1824c4ebccdedd9c0156fbd, entries=150, sequenceid=215, filesize=11.9 K 2024-12-07T18:14:57,500 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 0fc774415149ceeafbb0c8c46caa372c in 252ms, sequenceid=215, compaction requested=false 2024-12-07T18:14:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:57,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-07T18:14:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-07T18:14:57,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-07T18:14:57,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-12-07T18:14:57,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6490 sec 2024-12-07T18:14:57,731 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/6b5dd338a84847cb894fab57ff907fb2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6b5dd338a84847cb894fab57ff907fb2 2024-12-07T18:14:57,738 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/B of 0fc774415149ceeafbb0c8c46caa372c into 6b5dd338a84847cb894fab57ff907fb2(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:57,738 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:57,738 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/B, priority=11, startTime=1733595297192; duration=0sec 2024-12-07T18:14:57,738 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:57,738 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:B 2024-12-07T18:14:57,789 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/e7c638032f0f40c1b6d49d653ea64f50 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/e7c638032f0f40c1b6d49d653ea64f50 2024-12-07T18:14:57,795 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/C of 0fc774415149ceeafbb0c8c46caa372c into e7c638032f0f40c1b6d49d653ea64f50(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:57,795 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:57,795 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/C, priority=11, startTime=1733595297192; duration=0sec 2024-12-07T18:14:57,795 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:57,795 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:C 2024-12-07T18:14:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-07T18:14:58,009 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:14:58,010 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:14:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-07T18:14:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T18:14:58,014 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:14:58,016 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:14:58,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:14:58,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:14:58,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:58,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:58,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:58,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:58,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:58,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:14:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:58,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/44763859541f47258987b5ad4501ea05 is 50, key is test_row_0/A:col10/1733595298101/Put/seqid=0 2024-12-07T18:14:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T18:14:58,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742118_1294 (size=14541) 2024-12-07T18:14:58,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/44763859541f47258987b5ad4501ea05 2024-12-07T18:14:58,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595298143, max timestamp=1733595298156Num uniques 859; 2024-12-07T18:14:58,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:58,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595298143, max timestamp=1733595298156Num uniques 859; 2024-12-07T18:14:58,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:58,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/94ee575c08d24f51b90aa83d1bda285f is 50, key is test_row_0/B:col10/1733595298101/Put/seqid=0 2024-12-07T18:14:58,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595298143, max timestamp=1733595298156Num uniques 859; 2024-12-07T18:14:58,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:14:58,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-07T18:14:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742119_1295 (size=12151) 2024-12-07T18:14:58,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/94ee575c08d24f51b90aa83d1bda285f 2024-12-07T18:14:58,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/fe8db5886f39468ea047ea06b65f44e2 is 50, key is test_row_0/C:col10/1733595298101/Put/seqid=0 2024-12-07T18:14:58,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742120_1296 (size=12151) 2024-12-07T18:14:58,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595358238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595358238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595358243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595358247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595358249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-07T18:14:58,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:58,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T18:14:58,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595358359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595358359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595358359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595358359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595358359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-07T18:14:58,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:58,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595358568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595358569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595358569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595358569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595358569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/fe8db5886f39468ea047ea06b65f44e2 2024-12-07T18:14:58,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/44763859541f47258987b5ad4501ea05 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/44763859541f47258987b5ad4501ea05 2024-12-07T18:14:58,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/44763859541f47258987b5ad4501ea05, entries=200, sequenceid=228, filesize=14.2 K 2024-12-07T18:14:58,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/94ee575c08d24f51b90aa83d1bda285f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/94ee575c08d24f51b90aa83d1bda285f 2024-12-07T18:14:58,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/94ee575c08d24f51b90aa83d1bda285f, entries=150, sequenceid=228, filesize=11.9 K 2024-12-07T18:14:58,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/fe8db5886f39468ea047ea06b65f44e2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/fe8db5886f39468ea047ea06b65f44e2 2024-12-07T18:14:58,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-07T18:14:58,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:58,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:14:58,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/fe8db5886f39468ea047ea06b65f44e2, entries=150, sequenceid=228, filesize=11.9 K 2024-12-07T18:14:58,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0fc774415149ceeafbb0c8c46caa372c in 533ms, sequenceid=228, compaction requested=true 2024-12-07T18:14:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T18:14:58,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:14:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:14:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-07T18:14:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:14:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-07T18:14:58,639 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:58,639 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:58,640 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:58,640 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/A is initiating minor compaction (all files) 2024-12-07T18:14:58,640 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/A in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,641 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/eba085d67d75440e861df4a4a2e4428a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d7ee68fd188649599e0147e2d5169c2e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/44763859541f47258987b5ad4501ea05] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=38.3 K 2024-12-07T18:14:58,641 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:58,641 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/C is initiating minor compaction (all files) 2024-12-07T18:14:58,641 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/C in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,641 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/e7c638032f0f40c1b6d49d653ea64f50, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/04b8cb2dc1824c4ebccdedd9c0156fbd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/fe8db5886f39468ea047ea06b65f44e2] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.0 K 2024-12-07T18:14:58,641 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c638032f0f40c1b6d49d653ea64f50, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733595295772 2024-12-07T18:14:58,642 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting eba085d67d75440e861df4a4a2e4428a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733595295772 2024-12-07T18:14:58,642 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 04b8cb2dc1824c4ebccdedd9c0156fbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733595295905 2024-12-07T18:14:58,642 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7ee68fd188649599e0147e2d5169c2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733595295905 2024-12-07T18:14:58,642 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting fe8db5886f39468ea047ea06b65f44e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733595298098 2024-12-07T18:14:58,643 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44763859541f47258987b5ad4501ea05, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733595298090 2024-12-07T18:14:58,659 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#C#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:58,659 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/339088a84eac4f0aa502653b4c2f4cc5 is 50, key is test_row_0/C:col10/1733595298101/Put/seqid=0 2024-12-07T18:14:58,679 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:58,680 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/f8c0b2ffb4424bb59a5a863c361d3b6d is 50, key is test_row_0/A:col10/1733595298101/Put/seqid=0 2024-12-07T18:14:58,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742121_1297 (size=12629) 2024-12-07T18:14:58,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742122_1298 (size=12629) 2024-12-07T18:14:58,734 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/f8c0b2ffb4424bb59a5a863c361d3b6d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/f8c0b2ffb4424bb59a5a863c361d3b6d 2024-12-07T18:14:58,745 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/A of 0fc774415149ceeafbb0c8c46caa372c into f8c0b2ffb4424bb59a5a863c361d3b6d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:58,745 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:58,746 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/A, priority=13, startTime=1733595298639; duration=0sec 2024-12-07T18:14:58,746 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:14:58,746 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:A 2024-12-07T18:14:58,746 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:14:58,747 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:14:58,747 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/B is initiating minor compaction (all files) 2024-12-07T18:14:58,747 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/B in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,747 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6b5dd338a84847cb894fab57ff907fb2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/228d2e0f1b8d43a9b0a73c908444a270, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/94ee575c08d24f51b90aa83d1bda285f] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.0 K 2024-12-07T18:14:58,748 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b5dd338a84847cb894fab57ff907fb2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733595295772 2024-12-07T18:14:58,748 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 228d2e0f1b8d43a9b0a73c908444a270, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733595295905 2024-12-07T18:14:58,749 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94ee575c08d24f51b90aa83d1bda285f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733595298098 2024-12-07T18:14:58,761 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#B#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:14:58,761 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/0e363836899340fb925c34c01fc9774b is 50, key is test_row_0/B:col10/1733595298101/Put/seqid=0 2024-12-07T18:14:58,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742123_1299 (size=12629) 2024-12-07T18:14:58,776 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/0e363836899340fb925c34c01fc9774b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/0e363836899340fb925c34c01fc9774b 2024-12-07T18:14:58,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-07T18:14:58,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:58,788 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-07T18:14:58,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:14:58,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:58,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:14:58,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:58,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:14:58,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:14:58,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/6bc5f72ff0714f009aac33f06f8777a1 is 50, key is test_row_0/A:col10/1733595298237/Put/seqid=0 2024-12-07T18:14:58,798 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/B of 0fc774415149ceeafbb0c8c46caa372c into 0e363836899340fb925c34c01fc9774b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:58,798 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:58,798 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/B, priority=13, startTime=1733595298639; duration=0sec 2024-12-07T18:14:58,798 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:58,798 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:B 2024-12-07T18:14:58,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742124_1300 (size=12301) 2024-12-07T18:14:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:14:58,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:14:58,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595358889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595358895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595358896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595358896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595358897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:58,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595358998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595359009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595359009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595359009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595359010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,132 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/339088a84eac4f0aa502653b4c2f4cc5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/339088a84eac4f0aa502653b4c2f4cc5 2024-12-07T18:14:59,140 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/C of 0fc774415149ceeafbb0c8c46caa372c into 339088a84eac4f0aa502653b4c2f4cc5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:14:59,140 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:59,140 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/C, priority=13, startTime=1733595298639; duration=0sec 2024-12-07T18:14:59,140 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:14:59,140 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:C 2024-12-07T18:14:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T18:14:59,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595359208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,215 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/6bc5f72ff0714f009aac33f06f8777a1 2024-12-07T18:14:59,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595359219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595359219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595359219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,223 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595359219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/26dac45a61b54c76ab9bd4128abfb2c2 is 50, key is test_row_0/B:col10/1733595298237/Put/seqid=0 2024-12-07T18:14:59,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742125_1301 (size=12301) 2024-12-07T18:14:59,250 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/26dac45a61b54c76ab9bd4128abfb2c2 2024-12-07T18:14:59,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/6e623e9a38f648219fd590f70a2dbf56 is 50, key is test_row_0/C:col10/1733595298237/Put/seqid=0 2024-12-07T18:14:59,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742126_1302 (size=12301) 2024-12-07T18:14:59,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595359518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595359529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595359529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595359529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:14:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595359529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:14:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:14:59,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:14:59,668 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/6e623e9a38f648219fd590f70a2dbf56 2024-12-07T18:14:59,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/6bc5f72ff0714f009aac33f06f8777a1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/6bc5f72ff0714f009aac33f06f8777a1 2024-12-07T18:14:59,701 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/6bc5f72ff0714f009aac33f06f8777a1, entries=150, sequenceid=266, filesize=12.0 K 2024-12-07T18:14:59,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/26dac45a61b54c76ab9bd4128abfb2c2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/26dac45a61b54c76ab9bd4128abfb2c2 2024-12-07T18:14:59,708 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/26dac45a61b54c76ab9bd4128abfb2c2, entries=150, sequenceid=266, filesize=12.0 K 2024-12-07T18:14:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/6e623e9a38f648219fd590f70a2dbf56 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/6e623e9a38f648219fd590f70a2dbf56 2024-12-07T18:14:59,716 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/6e623e9a38f648219fd590f70a2dbf56, entries=150, sequenceid=266, filesize=12.0 K 2024-12-07T18:14:59,718 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 0fc774415149ceeafbb0c8c46caa372c in 930ms, sequenceid=266, compaction requested=false 2024-12-07T18:14:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:14:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:14:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-07T18:14:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-07T18:14:59,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-07T18:14:59,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7040 sec 2024-12-07T18:14:59,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.7140 sec 2024-12-07T18:15:00,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:00,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T18:15:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:00,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:00,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/3472330dab964cfd97e09df8e04e1b69 is 50, key is test_row_0/A:col10/1733595298893/Put/seqid=0 2024-12-07T18:15:00,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742127_1303 (size=12301) 2024-12-07T18:15:00,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/3472330dab964cfd97e09df8e04e1b69 2024-12-07T18:15:00,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/e5af8b34e28146e89e9d240f9fa8616f is 50, key is test_row_0/B:col10/1733595298893/Put/seqid=0 2024-12-07T18:15:00,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595360083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595360088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595360090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742128_1304 (size=12301) 2024-12-07T18:15:00,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595360091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595360093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-07T18:15:00,159 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:00,159 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-07T18:15:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-07T18:15:00,162 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:00,163 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:00,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:00,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595360198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595360198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595360198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595360209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595360209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-07T18:15:00,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-07T18:15:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:00,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:00,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595360409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595360409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595360409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595360418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595360418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-07T18:15:00,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:00,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:00,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:00,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:00,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-07T18:15:00,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/e5af8b34e28146e89e9d240f9fa8616f 2024-12-07T18:15:00,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/54f717cefd0f4098a5427e991ab1a814 is 50, key is test_row_0/C:col10/1733595298893/Put/seqid=0 2024-12-07T18:15:00,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742129_1305 (size=12301) 2024-12-07T18:15:00,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/54f717cefd0f4098a5427e991ab1a814 2024-12-07T18:15:00,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/3472330dab964cfd97e09df8e04e1b69 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3472330dab964cfd97e09df8e04e1b69 2024-12-07T18:15:00,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3472330dab964cfd97e09df8e04e1b69, entries=150, sequenceid=282, filesize=12.0 K 2024-12-07T18:15:00,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/e5af8b34e28146e89e9d240f9fa8616f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e5af8b34e28146e89e9d240f9fa8616f 2024-12-07T18:15:00,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e5af8b34e28146e89e9d240f9fa8616f, entries=150, sequenceid=282, filesize=12.0 K 2024-12-07T18:15:00,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/54f717cefd0f4098a5427e991ab1a814 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/54f717cefd0f4098a5427e991ab1a814 2024-12-07T18:15:00,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/54f717cefd0f4098a5427e991ab1a814, entries=150, sequenceid=282, filesize=12.0 K 2024-12-07T18:15:00,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0fc774415149ceeafbb0c8c46caa372c in 475ms, sequenceid=282, compaction requested=true 2024-12-07T18:15:00,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:00,540 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:00,540 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:00,541 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:00,541 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:00,541 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/B is initiating minor compaction (all files) 2024-12-07T18:15:00,541 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/A is initiating minor compaction (all files) 2024-12-07T18:15:00,541 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/A in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:00,541 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/B in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:00,541 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/f8c0b2ffb4424bb59a5a863c361d3b6d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/6bc5f72ff0714f009aac33f06f8777a1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3472330dab964cfd97e09df8e04e1b69] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.4 K 2024-12-07T18:15:00,541 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/0e363836899340fb925c34c01fc9774b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/26dac45a61b54c76ab9bd4128abfb2c2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e5af8b34e28146e89e9d240f9fa8616f] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.4 K 2024-12-07T18:15:00,541 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8c0b2ffb4424bb59a5a863c361d3b6d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733595298098 2024-12-07T18:15:00,542 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e363836899340fb925c34c01fc9774b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733595298098 2024-12-07T18:15:00,542 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 26dac45a61b54c76ab9bd4128abfb2c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733595298191 2024-12-07T18:15:00,542 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bc5f72ff0714f009aac33f06f8777a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733595298191 2024-12-07T18:15:00,543 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e5af8b34e28146e89e9d240f9fa8616f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733595298893 2024-12-07T18:15:00,543 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3472330dab964cfd97e09df8e04e1b69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733595298893 2024-12-07T18:15:00,556 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:00,557 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#A#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:00,557 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/9692467403414cdd84cca744a35ed207 is 50, key is test_row_0/B:col10/1733595298893/Put/seqid=0 2024-12-07T18:15:00,557 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/c7e91fbfd0c14ec68e615e9662e28757 is 50, key is test_row_0/A:col10/1733595298893/Put/seqid=0 2024-12-07T18:15:00,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742130_1306 (size=12881) 2024-12-07T18:15:00,573 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/9692467403414cdd84cca744a35ed207 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/9692467403414cdd84cca744a35ed207 2024-12-07T18:15:00,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742131_1307 (size=12881) 2024-12-07T18:15:00,582 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/B of 0fc774415149ceeafbb0c8c46caa372c into 9692467403414cdd84cca744a35ed207(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:00,582 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:00,582 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/B, priority=13, startTime=1733595300540; duration=0sec 2024-12-07T18:15:00,582 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:00,582 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:B 2024-12-07T18:15:00,583 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:00,585 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:00,585 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/C is initiating minor compaction (all files) 2024-12-07T18:15:00,585 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/C in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:00,585 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/339088a84eac4f0aa502653b4c2f4cc5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/6e623e9a38f648219fd590f70a2dbf56, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/54f717cefd0f4098a5427e991ab1a814] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.4 K 2024-12-07T18:15:00,585 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 339088a84eac4f0aa502653b4c2f4cc5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1733595298098 2024-12-07T18:15:00,586 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e623e9a38f648219fd590f70a2dbf56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733595298191 2024-12-07T18:15:00,586 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f717cefd0f4098a5427e991ab1a814, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733595298893 2024-12-07T18:15:00,592 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:00,593 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/ef842b7aa99748a3a46029787db77b04 is 50, key is test_row_0/C:col10/1733595298893/Put/seqid=0 2024-12-07T18:15:00,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742132_1308 (size=12881) 2024-12-07T18:15:00,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-07T18:15:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:00,623 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T18:15:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:00,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:00,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/3d9ad2aeabec442892421e18f296aa9f is 50, key is test_row_0/A:col10/1733595300088/Put/seqid=0 2024-12-07T18:15:00,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742133_1309 (size=12301) 2024-12-07T18:15:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:00,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:00,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595360735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595360737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595360740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595360746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595360741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-07T18:15:00,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595360849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595360849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595360858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595360858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595360858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:00,980 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/c7e91fbfd0c14ec68e615e9662e28757 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/c7e91fbfd0c14ec68e615e9662e28757 2024-12-07T18:15:00,986 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/A of 0fc774415149ceeafbb0c8c46caa372c into c7e91fbfd0c14ec68e615e9662e28757(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:00,986 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:00,986 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/A, priority=13, startTime=1733595300540; duration=0sec 2024-12-07T18:15:00,987 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:00,987 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:A 2024-12-07T18:15:01,003 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/ef842b7aa99748a3a46029787db77b04 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ef842b7aa99748a3a46029787db77b04 2024-12-07T18:15:01,009 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/C of 0fc774415149ceeafbb0c8c46caa372c into ef842b7aa99748a3a46029787db77b04(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:01,009 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:01,009 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/C, priority=13, startTime=1733595300540; duration=0sec 2024-12-07T18:15:01,009 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:01,009 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:C 2024-12-07T18:15:01,046 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/3d9ad2aeabec442892421e18f296aa9f 2024-12-07T18:15:01,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/6f37daa98d04424c9a7f73b3b23c126c is 50, key is test_row_0/B:col10/1733595300088/Put/seqid=0 2024-12-07T18:15:01,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595361058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595361058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742134_1310 (size=12301) 2024-12-07T18:15:01,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595361068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595361068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595361068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,071 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/6f37daa98d04424c9a7f73b3b23c126c 2024-12-07T18:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/4452f5ce14124efc986226474e652727 is 50, key is test_row_0/C:col10/1733595300088/Put/seqid=0 2024-12-07T18:15:01,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742135_1311 (size=12301) 2024-12-07T18:15:01,095 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/4452f5ce14124efc986226474e652727 2024-12-07T18:15:01,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/3d9ad2aeabec442892421e18f296aa9f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3d9ad2aeabec442892421e18f296aa9f 2024-12-07T18:15:01,108 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3d9ad2aeabec442892421e18f296aa9f, entries=150, sequenceid=301, filesize=12.0 K 2024-12-07T18:15:01,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/6f37daa98d04424c9a7f73b3b23c126c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6f37daa98d04424c9a7f73b3b23c126c 2024-12-07T18:15:01,118 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6f37daa98d04424c9a7f73b3b23c126c, entries=150, sequenceid=301, filesize=12.0 K 2024-12-07T18:15:01,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/4452f5ce14124efc986226474e652727 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4452f5ce14124efc986226474e652727 2024-12-07T18:15:01,127 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4452f5ce14124efc986226474e652727, entries=150, sequenceid=301, filesize=12.0 K 2024-12-07T18:15:01,129 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 0fc774415149ceeafbb0c8c46caa372c in 506ms, sequenceid=301, compaction requested=false 2024-12-07T18:15:01,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:01,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:01,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-07T18:15:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-07T18:15:01,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-07T18:15:01,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 968 msec 2024-12-07T18:15:01,137 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 976 msec 2024-12-07T18:15:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-07T18:15:01,299 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:01,300 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:01,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-07T18:15:01,302 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T18:15:01,303 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:01,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:01,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:01,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T18:15:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:01,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:01,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/5f29bbc174ab4a0dacbd376914693598 is 50, key is test_row_0/A:col10/1733595301369/Put/seqid=0 2024-12-07T18:15:01,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742136_1312 (size=14741) 2024-12-07T18:15:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T18:15:01,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595361439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595361440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595361440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595361448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595361450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-07T18:15:01,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:01,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:01,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:01,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:01,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595361558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595361558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595361558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595361558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595361558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-07T18:15:01,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:01,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:01,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:01,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:01,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T18:15:01,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-07T18:15:01,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:01,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:01,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:01,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:01,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595361768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595361768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595361769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595361769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:01,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595361769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:01,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:01,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/5f29bbc174ab4a0dacbd376914693598 2024-12-07T18:15:01,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/634a17a3683f4246b9f8c08ecb8f5d04 is 50, key is test_row_0/B:col10/1733595301369/Put/seqid=0 2024-12-07T18:15:01,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742137_1313 (size=12301) 2024-12-07T18:15:01,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/634a17a3683f4246b9f8c08ecb8f5d04 2024-12-07T18:15:01,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/4c989033878840d499d286ae8accd201 is 50, key is test_row_0/C:col10/1733595301369/Put/seqid=0 2024-12-07T18:15:01,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742138_1314 (size=12301) 2024-12-07T18:15:01,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/4c989033878840d499d286ae8accd201 2024-12-07T18:15:01,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/5f29bbc174ab4a0dacbd376914693598 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/5f29bbc174ab4a0dacbd376914693598 2024-12-07T18:15:01,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/5f29bbc174ab4a0dacbd376914693598, entries=200, sequenceid=320, filesize=14.4 K 2024-12-07T18:15:01,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/634a17a3683f4246b9f8c08ecb8f5d04 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/634a17a3683f4246b9f8c08ecb8f5d04 2024-12-07T18:15:01,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/634a17a3683f4246b9f8c08ecb8f5d04, entries=150, sequenceid=320, filesize=12.0 K 2024-12-07T18:15:01,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/4c989033878840d499d286ae8accd201 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4c989033878840d499d286ae8accd201 2024-12-07T18:15:01,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4c989033878840d499d286ae8accd201, entries=150, sequenceid=320, filesize=12.0 K 2024-12-07T18:15:01,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0fc774415149ceeafbb0c8c46caa372c in 522ms, sequenceid=320, compaction requested=true 2024-12-07T18:15:01,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:01,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:01,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:01,893 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:01,893 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:01,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:01,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:01,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:01,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:01,896 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:01,896 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:01,896 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/B is initiating minor compaction (all files) 2024-12-07T18:15:01,896 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/A is initiating minor compaction (all files) 2024-12-07T18:15:01,896 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/B in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:01,896 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/A in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:01,897 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/9692467403414cdd84cca744a35ed207, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6f37daa98d04424c9a7f73b3b23c126c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/634a17a3683f4246b9f8c08ecb8f5d04] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.6 K 2024-12-07T18:15:01,897 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/c7e91fbfd0c14ec68e615e9662e28757, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3d9ad2aeabec442892421e18f296aa9f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/5f29bbc174ab4a0dacbd376914693598] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=39.0 K 2024-12-07T18:15:01,897 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 9692467403414cdd84cca744a35ed207, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733595298893 2024-12-07T18:15:01,897 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7e91fbfd0c14ec68e615e9662e28757, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733595298893 2024-12-07T18:15:01,898 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f37daa98d04424c9a7f73b3b23c126c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733595300088 2024-12-07T18:15:01,898 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d9ad2aeabec442892421e18f296aa9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733595300088 2024-12-07T18:15:01,898 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f29bbc174ab4a0dacbd376914693598, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733595300733 2024-12-07T18:15:01,898 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 634a17a3683f4246b9f8c08ecb8f5d04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733595300737 2024-12-07T18:15:01,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-07T18:15:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:01,926 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T18:15:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:01,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:01,928 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:01,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T18:15:01,928 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/6cb1a1a08e65463eb695f2d4b56f3af9 is 50, key is test_row_0/B:col10/1733595301369/Put/seqid=0 2024-12-07T18:15:01,943 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#A#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:01,943 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/843aec477b1642ba9d7806c505069aae is 50, key is test_row_0/A:col10/1733595301369/Put/seqid=0 2024-12-07T18:15:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/69456a031889430880e3e5665712b891 is 50, key is test_row_0/A:col10/1733595301386/Put/seqid=0 2024-12-07T18:15:01,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742139_1315 (size=12983) 2024-12-07T18:15:02,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742140_1316 (size=12983) 2024-12-07T18:15:02,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742141_1317 (size=12301) 2024-12-07T18:15:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:02,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:02,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595362102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595362104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595362102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595362105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595362106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595362218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595362218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595362219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595362219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595362219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,403 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/6cb1a1a08e65463eb695f2d4b56f3af9 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6cb1a1a08e65463eb695f2d4b56f3af9 2024-12-07T18:15:02,413 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/B of 0fc774415149ceeafbb0c8c46caa372c into 6cb1a1a08e65463eb695f2d4b56f3af9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:02,413 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:02,413 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/B, priority=13, startTime=1733595301893; duration=0sec 2024-12-07T18:15:02,413 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:02,413 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:B 2024-12-07T18:15:02,413 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:02,416 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:02,416 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/C is initiating minor compaction (all files) 2024-12-07T18:15:02,416 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/C in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:02,417 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ef842b7aa99748a3a46029787db77b04, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4452f5ce14124efc986226474e652727, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4c989033878840d499d286ae8accd201] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.6 K 2024-12-07T18:15:02,418 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ef842b7aa99748a3a46029787db77b04, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733595298893 2024-12-07T18:15:02,419 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 4452f5ce14124efc986226474e652727, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733595300088 2024-12-07T18:15:02,420 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c989033878840d499d286ae8accd201, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733595300737 2024-12-07T18:15:02,421 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/69456a031889430880e3e5665712b891 2024-12-07T18:15:02,427 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/843aec477b1642ba9d7806c505069aae as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/843aec477b1642ba9d7806c505069aae 2024-12-07T18:15:02,437 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/A of 0fc774415149ceeafbb0c8c46caa372c into 843aec477b1642ba9d7806c505069aae(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:02,437 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:02,437 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/A, priority=13, startTime=1733595301893; duration=0sec 2024-12-07T18:15:02,438 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:02,438 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:A 2024-12-07T18:15:02,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/ab91ce678f414079ac6d46f110baedf2 is 50, key is test_row_0/B:col10/1733595301386/Put/seqid=0 2024-12-07T18:15:02,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T18:15:02,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595362439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,443 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,446 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595362439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595362439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,446 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/64a0dd9eb9114e03ac4536964fd0a932 is 50, key is test_row_0/C:col10/1733595301369/Put/seqid=0 2024-12-07T18:15:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595362440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595362440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742142_1318 (size=12301) 2024-12-07T18:15:02,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742143_1319 (size=12983) 2024-12-07T18:15:02,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595362748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595362758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595362758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595362758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595362758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:02,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:02,865 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/ab91ce678f414079ac6d46f110baedf2 2024-12-07T18:15:02,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/2433079dd0b94a6dbe4b1e9a4a558a23 is 50, key is test_row_0/C:col10/1733595301386/Put/seqid=0 2024-12-07T18:15:02,884 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/64a0dd9eb9114e03ac4536964fd0a932 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/64a0dd9eb9114e03ac4536964fd0a932 2024-12-07T18:15:02,891 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/C of 0fc774415149ceeafbb0c8c46caa372c into 64a0dd9eb9114e03ac4536964fd0a932(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:02,891 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:02,891 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/C, priority=13, startTime=1733595301894; duration=0sec 2024-12-07T18:15:02,891 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:02,891 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:C 2024-12-07T18:15:02,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742144_1320 (size=12301) 2024-12-07T18:15:02,898 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/2433079dd0b94a6dbe4b1e9a4a558a23 2024-12-07T18:15:02,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/69456a031889430880e3e5665712b891 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69456a031889430880e3e5665712b891 2024-12-07T18:15:02,909 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69456a031889430880e3e5665712b891, entries=150, sequenceid=335, filesize=12.0 K 2024-12-07T18:15:02,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/ab91ce678f414079ac6d46f110baedf2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/ab91ce678f414079ac6d46f110baedf2 2024-12-07T18:15:02,916 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/ab91ce678f414079ac6d46f110baedf2, entries=150, sequenceid=335, filesize=12.0 K 2024-12-07T18:15:02,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/2433079dd0b94a6dbe4b1e9a4a558a23 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/2433079dd0b94a6dbe4b1e9a4a558a23 2024-12-07T18:15:02,923 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/2433079dd0b94a6dbe4b1e9a4a558a23, entries=150, sequenceid=335, filesize=12.0 K 2024-12-07T18:15:02,925 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 0fc774415149ceeafbb0c8c46caa372c in 999ms, sequenceid=335, compaction requested=false 2024-12-07T18:15:02,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:02,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:02,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-07T18:15:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-07T18:15:02,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-07T18:15:02,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6250 sec 2024-12-07T18:15:02,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6300 sec 2024-12-07T18:15:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:03,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T18:15:03,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:03,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:03,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:03,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:03,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:03,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:03,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/69335f8d9b3441319ad0580668732423 is 50, key is test_row_0/A:col10/1733595302102/Put/seqid=0 2024-12-07T18:15:03,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595363280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595363281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595363281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595363283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595363283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742145_1321 (size=14741) 2024-12-07T18:15:03,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595363388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595363398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595363398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595363398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595363399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-07T18:15:03,448 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:03,449 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-07T18:15:03,451 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T18:15:03,452 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:03,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:03,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T18:15:03,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595363599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-07T18:15:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:03,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:03,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595363608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595363608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595363608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595363609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/69335f8d9b3441319ad0580668732423 2024-12-07T18:15:03,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/e9b5951821724f85b437197ad5c517ca is 50, key is test_row_0/B:col10/1733595302102/Put/seqid=0 2024-12-07T18:15:03,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742146_1322 (size=12301) 2024-12-07T18:15:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T18:15:03,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-07T18:15:03,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:03,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:03,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:03,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:03,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:03,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595363908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595363918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-07T18:15:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595363918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:03,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:03,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:03,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595363918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:03,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:03,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595363929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:03,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:04,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T18:15:04,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-07T18:15:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/e9b5951821724f85b437197ad5c517ca 2024-12-07T18:15:04,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/c1b3551ce5e34bddbb3e0be1ff177bf4 is 50, key is test_row_0/C:col10/1733595302102/Put/seqid=0 2024-12-07T18:15:04,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742147_1323 (size=12301) 2024-12-07T18:15:04,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-07T18:15:04,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-07T18:15:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595364418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:04,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:04,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595364428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:04,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595364428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:04,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:04,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595364428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:04,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595364439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:04,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:04,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-07T18:15:04,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:04,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:04,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/c1b3551ce5e34bddbb3e0be1ff177bf4 2024-12-07T18:15:04,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/69335f8d9b3441319ad0580668732423 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69335f8d9b3441319ad0580668732423 2024-12-07T18:15:04,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69335f8d9b3441319ad0580668732423, entries=200, sequenceid=357, filesize=14.4 K 2024-12-07T18:15:04,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/e9b5951821724f85b437197ad5c517ca as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e9b5951821724f85b437197ad5c517ca 2024-12-07T18:15:04,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e9b5951821724f85b437197ad5c517ca, entries=150, sequenceid=357, filesize=12.0 K 2024-12-07T18:15:04,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/c1b3551ce5e34bddbb3e0be1ff177bf4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c1b3551ce5e34bddbb3e0be1ff177bf4 2024-12-07T18:15:04,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c1b3551ce5e34bddbb3e0be1ff177bf4, entries=150, sequenceid=357, filesize=12.0 K 2024-12-07T18:15:04,575 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0fc774415149ceeafbb0c8c46caa372c in 1313ms, sequenceid=357, compaction requested=true 2024-12-07T18:15:04,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:04,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:04,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:04,575 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:04,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:04,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:04,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:04,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:04,575 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:04,576 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:04,576 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:04,576 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/B is initiating minor compaction (all files) 2024-12-07T18:15:04,576 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/A is initiating minor compaction (all files) 2024-12-07T18:15:04,576 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/B in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,576 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/A in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,576 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6cb1a1a08e65463eb695f2d4b56f3af9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/ab91ce678f414079ac6d46f110baedf2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e9b5951821724f85b437197ad5c517ca] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.7 K 2024-12-07T18:15:04,576 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/843aec477b1642ba9d7806c505069aae, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69456a031889430880e3e5665712b891, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69335f8d9b3441319ad0580668732423] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=39.1 K 2024-12-07T18:15:04,577 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb1a1a08e65463eb695f2d4b56f3af9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733595300737 2024-12-07T18:15:04,577 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 843aec477b1642ba9d7806c505069aae, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733595300737 2024-12-07T18:15:04,577 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ab91ce678f414079ac6d46f110baedf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733595301386 2024-12-07T18:15:04,577 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69456a031889430880e3e5665712b891, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733595301386 2024-12-07T18:15:04,578 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69335f8d9b3441319ad0580668732423, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733595302100 2024-12-07T18:15:04,579 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e9b5951821724f85b437197ad5c517ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733595302102 2024-12-07T18:15:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T18:15:04,595 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:04,595 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/9bd8a1804b654cc19dc158d81cdd1865 is 50, key is test_row_0/A:col10/1733595302102/Put/seqid=0 2024-12-07T18:15:04,603 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:04,603 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/9cf6da06eb49488ea0cd180d123539b7 is 50, key is test_row_0/B:col10/1733595302102/Put/seqid=0 2024-12-07T18:15:04,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742148_1324 (size=13085) 2024-12-07T18:15:04,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742149_1325 (size=13085) 2024-12-07T18:15:04,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-07T18:15:04,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:04,693 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T18:15:04,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:04,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:04,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:04,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:04,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:04,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:04,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/4997235d7bf4441b9173fafa76395595 is 50, key is test_row_0/A:col10/1733595303278/Put/seqid=0 2024-12-07T18:15:04,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742150_1326 (size=12301) 2024-12-07T18:15:05,029 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/9bd8a1804b654cc19dc158d81cdd1865 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/9bd8a1804b654cc19dc158d81cdd1865 2024-12-07T18:15:05,035 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/A of 0fc774415149ceeafbb0c8c46caa372c into 9bd8a1804b654cc19dc158d81cdd1865(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:05,035 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:05,035 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/A, priority=13, startTime=1733595304575; duration=0sec 2024-12-07T18:15:05,035 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:05,035 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:A 2024-12-07T18:15:05,035 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:05,036 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:05,036 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/C is initiating minor compaction (all files) 2024-12-07T18:15:05,037 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/C in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:05,037 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/64a0dd9eb9114e03ac4536964fd0a932, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/2433079dd0b94a6dbe4b1e9a4a558a23, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c1b3551ce5e34bddbb3e0be1ff177bf4] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.7 K 2024-12-07T18:15:05,037 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64a0dd9eb9114e03ac4536964fd0a932, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1733595300737 2024-12-07T18:15:05,038 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2433079dd0b94a6dbe4b1e9a4a558a23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733595301386 2024-12-07T18:15:05,038 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1b3551ce5e34bddbb3e0be1ff177bf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733595302102 2024-12-07T18:15:05,043 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/9cf6da06eb49488ea0cd180d123539b7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/9cf6da06eb49488ea0cd180d123539b7 2024-12-07T18:15:05,049 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#C#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:05,050 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/22b96980aad34dac92972103ff9b2afe is 50, key is test_row_0/C:col10/1733595302102/Put/seqid=0 2024-12-07T18:15:05,052 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/B of 0fc774415149ceeafbb0c8c46caa372c into 9cf6da06eb49488ea0cd180d123539b7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:05,052 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:05,052 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/B, priority=13, startTime=1733595304575; duration=0sec 2024-12-07T18:15:05,052 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:05,052 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:B 2024-12-07T18:15:05,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742151_1327 (size=13085) 2024-12-07T18:15:05,068 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/22b96980aad34dac92972103ff9b2afe as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/22b96980aad34dac92972103ff9b2afe 2024-12-07T18:15:05,074 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/C of 0fc774415149ceeafbb0c8c46caa372c into 22b96980aad34dac92972103ff9b2afe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:05,074 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:05,074 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/C, priority=13, startTime=1733595304575; duration=0sec 2024-12-07T18:15:05,075 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:05,075 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:C 2024-12-07T18:15:05,107 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/4997235d7bf4441b9173fafa76395595 2024-12-07T18:15:05,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/69120230782c448dad2fa112e3dd1fb0 is 50, key is test_row_0/B:col10/1733595303278/Put/seqid=0 2024-12-07T18:15:05,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742152_1328 (size=12301) 2024-12-07T18:15:05,123 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/69120230782c448dad2fa112e3dd1fb0 2024-12-07T18:15:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/068f1d09d8f14f85b711dc8a422cdf6a is 50, key is test_row_0/C:col10/1733595303278/Put/seqid=0 2024-12-07T18:15:05,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742153_1329 (size=12301) 2024-12-07T18:15:05,142 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/068f1d09d8f14f85b711dc8a422cdf6a 2024-12-07T18:15:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/4997235d7bf4441b9173fafa76395595 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4997235d7bf4441b9173fafa76395595 2024-12-07T18:15:05,155 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4997235d7bf4441b9173fafa76395595, entries=150, sequenceid=370, filesize=12.0 K 2024-12-07T18:15:05,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/69120230782c448dad2fa112e3dd1fb0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/69120230782c448dad2fa112e3dd1fb0 2024-12-07T18:15:05,162 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/69120230782c448dad2fa112e3dd1fb0, entries=150, sequenceid=370, filesize=12.0 K 2024-12-07T18:15:05,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/068f1d09d8f14f85b711dc8a422cdf6a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/068f1d09d8f14f85b711dc8a422cdf6a 2024-12-07T18:15:05,168 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/068f1d09d8f14f85b711dc8a422cdf6a, entries=150, sequenceid=370, filesize=12.0 K 2024-12-07T18:15:05,169 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 0fc774415149ceeafbb0c8c46caa372c in 476ms, sequenceid=370, compaction requested=false 2024-12-07T18:15:05,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:05,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:05,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-07T18:15:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-07T18:15:05,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-07T18:15:05,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-12-07T18:15:05,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.7240 sec 2024-12-07T18:15:05,435 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T18:15:05,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:15:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:05,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:05,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:05,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:05,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:05,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:05,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:05,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/b04f09bfdc1b47a19c887ea92c53b810 is 50, key is test_row_0/A:col10/1733595305479/Put/seqid=0 2024-12-07T18:15:05,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595365530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595365530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595365531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595365533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742154_1330 (size=12301) 2024-12-07T18:15:05,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/b04f09bfdc1b47a19c887ea92c53b810 2024-12-07T18:15:05,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595365538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/f4d6e02d6ec348279a6c7e05960ed8db is 50, key is test_row_0/B:col10/1733595305479/Put/seqid=0 2024-12-07T18:15:05,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742155_1331 (size=12301) 2024-12-07T18:15:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-07T18:15:05,599 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:05,599 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-07T18:15:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T18:15:05,603 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:05,604 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:05,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:05,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595365648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595365648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595365648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595365648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595365648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T18:15:05,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T18:15:05,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:05,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:05,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:05,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595365858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595365859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595365859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,863 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595365859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595365860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T18:15:05,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T18:15:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:05,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:05,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/f4d6e02d6ec348279a6c7e05960ed8db 2024-12-07T18:15:05,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/ea4ae9b36efc4039897db6885159b1c1 is 50, key is test_row_0/C:col10/1733595305479/Put/seqid=0 2024-12-07T18:15:06,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742156_1332 (size=12301) 2024-12-07T18:15:06,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T18:15:06,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:06,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:06,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:06,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:06,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:06,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595366169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595366169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595366169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595366169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595366169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T18:15:06,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T18:15:06,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:06,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:06,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:06,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:06,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T18:15:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:06,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:06,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:06,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:06,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/ea4ae9b36efc4039897db6885159b1c1 2024-12-07T18:15:06,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/b04f09bfdc1b47a19c887ea92c53b810 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/b04f09bfdc1b47a19c887ea92c53b810 2024-12-07T18:15:06,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/b04f09bfdc1b47a19c887ea92c53b810, entries=150, sequenceid=385, filesize=12.0 K 2024-12-07T18:15:06,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/f4d6e02d6ec348279a6c7e05960ed8db as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/f4d6e02d6ec348279a6c7e05960ed8db 2024-12-07T18:15:06,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/f4d6e02d6ec348279a6c7e05960ed8db, entries=150, sequenceid=385, filesize=12.0 K 2024-12-07T18:15:06,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/ea4ae9b36efc4039897db6885159b1c1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea4ae9b36efc4039897db6885159b1c1 2024-12-07T18:15:06,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea4ae9b36efc4039897db6885159b1c1, entries=150, sequenceid=385, filesize=12.0 K 2024-12-07T18:15:06,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0fc774415149ceeafbb0c8c46caa372c in 955ms, sequenceid=385, compaction requested=true 2024-12-07T18:15:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:06,437 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:06,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:06,437 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:06,438 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:06,438 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/A is initiating minor compaction (all files) 2024-12-07T18:15:06,439 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/A in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:06,439 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/9bd8a1804b654cc19dc158d81cdd1865, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4997235d7bf4441b9173fafa76395595, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/b04f09bfdc1b47a19c887ea92c53b810] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.8 K 2024-12-07T18:15:06,439 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd8a1804b654cc19dc158d81cdd1865, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733595302102 2024-12-07T18:15:06,440 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:06,440 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/B is initiating minor compaction (all files) 2024-12-07T18:15:06,440 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4997235d7bf4441b9173fafa76395595, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733595303278 2024-12-07T18:15:06,440 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/B in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:06,440 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/9cf6da06eb49488ea0cd180d123539b7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/69120230782c448dad2fa112e3dd1fb0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/f4d6e02d6ec348279a6c7e05960ed8db] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.8 K 2024-12-07T18:15:06,440 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting b04f09bfdc1b47a19c887ea92c53b810, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733595305439 2024-12-07T18:15:06,440 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf6da06eb49488ea0cd180d123539b7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733595302102 2024-12-07T18:15:06,444 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 69120230782c448dad2fa112e3dd1fb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733595303278 2024-12-07T18:15:06,445 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d6e02d6ec348279a6c7e05960ed8db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733595305439 2024-12-07T18:15:06,451 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#A#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:06,451 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/4a46928a08c04cd582c1c94ce87cbc4f is 50, key is test_row_0/A:col10/1733595305479/Put/seqid=0 2024-12-07T18:15:06,455 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:06,456 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/a73c78a9774d45f5a92156b91038823f is 50, key is test_row_0/B:col10/1733595305479/Put/seqid=0 2024-12-07T18:15:06,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742157_1333 (size=13187) 2024-12-07T18:15:06,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742158_1334 (size=13187) 2024-12-07T18:15:06,483 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/a73c78a9774d45f5a92156b91038823f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a73c78a9774d45f5a92156b91038823f 2024-12-07T18:15:06,489 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/B of 0fc774415149ceeafbb0c8c46caa372c into a73c78a9774d45f5a92156b91038823f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:06,489 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:06,489 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/B, priority=13, startTime=1733595306437; duration=0sec 2024-12-07T18:15:06,490 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:06,490 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:B 2024-12-07T18:15:06,490 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:06,491 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:06,491 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/C is initiating minor compaction (all files) 2024-12-07T18:15:06,491 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/C in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:06,491 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/22b96980aad34dac92972103ff9b2afe, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/068f1d09d8f14f85b711dc8a422cdf6a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea4ae9b36efc4039897db6885159b1c1] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.8 K 2024-12-07T18:15:06,492 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b96980aad34dac92972103ff9b2afe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1733595302102 2024-12-07T18:15:06,492 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 068f1d09d8f14f85b711dc8a422cdf6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733595303278 2024-12-07T18:15:06,493 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ea4ae9b36efc4039897db6885159b1c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733595305439 2024-12-07T18:15:06,511 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:06,511 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/28b2de4087d34df8b284f193786a7ecf is 50, key is test_row_0/C:col10/1733595305479/Put/seqid=0 2024-12-07T18:15:06,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742159_1335 (size=13187) 2024-12-07T18:15:06,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-07T18:15:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:06,536 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T18:15:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:06,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:06,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/254dfa0c327b4f7bbfd39720d014f352 is 50, key is test_row_0/A:col10/1733595305531/Put/seqid=0 2024-12-07T18:15:06,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742160_1336 (size=12301) 2024-12-07T18:15:06,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:06,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:06,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595366688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595366689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595366690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595366691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595366692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T18:15:06,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595366798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595366798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595366798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595366798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595366798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:06,872 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/4a46928a08c04cd582c1c94ce87cbc4f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4a46928a08c04cd582c1c94ce87cbc4f 2024-12-07T18:15:06,876 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/A of 0fc774415149ceeafbb0c8c46caa372c into 4a46928a08c04cd582c1c94ce87cbc4f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:06,877 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:06,877 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/A, priority=13, startTime=1733595306437; duration=0sec 2024-12-07T18:15:06,877 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:06,877 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:A 2024-12-07T18:15:06,937 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/28b2de4087d34df8b284f193786a7ecf as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/28b2de4087d34df8b284f193786a7ecf 2024-12-07T18:15:06,944 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/C of 0fc774415149ceeafbb0c8c46caa372c into 28b2de4087d34df8b284f193786a7ecf(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:06,944 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:06,944 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/C, priority=13, startTime=1733595306437; duration=0sec 2024-12-07T18:15:06,944 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:06,944 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:C 2024-12-07T18:15:06,951 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/254dfa0c327b4f7bbfd39720d014f352 2024-12-07T18:15:06,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/3f5b569d1aa14fe89af47a549a38abc5 is 50, key is test_row_0/B:col10/1733595305531/Put/seqid=0 2024-12-07T18:15:06,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742161_1337 (size=12301) 2024-12-07T18:15:07,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595367008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595367008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595367008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595367008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595367009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595367318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595367318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595367318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595367318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595367318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,371 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/3f5b569d1aa14fe89af47a549a38abc5 2024-12-07T18:15:07,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/5b018c03da6d4ab1b6e00be1c2449c3e is 50, key is test_row_0/C:col10/1733595305531/Put/seqid=0 2024-12-07T18:15:07,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742162_1338 (size=12301) 2024-12-07T18:15:07,383 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/5b018c03da6d4ab1b6e00be1c2449c3e 2024-12-07T18:15:07,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/254dfa0c327b4f7bbfd39720d014f352 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/254dfa0c327b4f7bbfd39720d014f352 2024-12-07T18:15:07,394 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/254dfa0c327b4f7bbfd39720d014f352, entries=150, sequenceid=406, filesize=12.0 K 2024-12-07T18:15:07,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/3f5b569d1aa14fe89af47a549a38abc5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3f5b569d1aa14fe89af47a549a38abc5 2024-12-07T18:15:07,398 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3f5b569d1aa14fe89af47a549a38abc5, entries=150, sequenceid=406, filesize=12.0 K 2024-12-07T18:15:07,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/5b018c03da6d4ab1b6e00be1c2449c3e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/5b018c03da6d4ab1b6e00be1c2449c3e 2024-12-07T18:15:07,403 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/5b018c03da6d4ab1b6e00be1c2449c3e, entries=150, sequenceid=406, filesize=12.0 K 2024-12-07T18:15:07,403 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0fc774415149ceeafbb0c8c46caa372c in 867ms, sequenceid=406, compaction requested=false 2024-12-07T18:15:07,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:07,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:07,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-07T18:15:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-07T18:15:07,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-07T18:15:07,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8010 sec 2024-12-07T18:15:07,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.8080 sec 2024-12-07T18:15:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-07T18:15:07,749 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:07,749 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-07T18:15:07,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T18:15:07,751 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:07,752 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:07,752 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:07,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T18:15:07,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:07,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:07,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:07,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:07,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:07,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:07,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/318beef7128a4499af1500af5ebc2765 is 50, key is test_row_0/A:col10/1733595306691/Put/seqid=0 2024-12-07T18:15:07,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742163_1339 (size=12301) 2024-12-07T18:15:07,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595367848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595367848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595367849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595367854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T18:15:07,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595367855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T18:15:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:07,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:07,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:07,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595367959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595367959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595367959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595367968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595367968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,058 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T18:15:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T18:15:08,079 INFO [Thread-1189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1189. 2024-12-07T18:15:08,079 DEBUG [Thread-1189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:08,079 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:08,080 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:08,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:08,081 INFO [Thread-1193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1193. 2024-12-07T18:15:08,081 DEBUG [Thread-1193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:08,081 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:08,081 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:08,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:08,082 INFO [Thread-1197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1197. 2024-12-07T18:15:08,082 DEBUG [Thread-1197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:08,082 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:08,082 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:08,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:08,084 INFO [Thread-1195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1195. 2024-12-07T18:15:08,084 DEBUG [Thread-1195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:08,084 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:08,084 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:08,084 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:08,085 INFO [Thread-1191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1191. 2024-12-07T18:15:08,085 DEBUG [Thread-1191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:08,085 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:08,085 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:08,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:08,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595368168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595368168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595368168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595368178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595368178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T18:15:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/318beef7128a4499af1500af5ebc2765 2024-12-07T18:15:08,249 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/243b5de1bd904a3ca358929be3f66be7 is 50, key is test_row_0/B:col10/1733595306691/Put/seqid=0 2024-12-07T18:15:08,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742164_1340 (size=12301) 2024-12-07T18:15:08,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T18:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T18:15:08,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595368479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595368479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595368478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595368488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595368488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T18:15:08,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:08,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/243b5de1bd904a3ca358929be3f66be7 2024-12-07T18:15:08,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/23533b5e80cc49c29bd40bbffa9308f1 is 50, key is test_row_0/C:col10/1733595306691/Put/seqid=0 2024-12-07T18:15:08,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742165_1341 (size=12301) 2024-12-07T18:15:08,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T18:15:08,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:08,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T18:15:08,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:08,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T18:15:08,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T18:15:08,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:08,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:08,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:08,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49744 deadline: 1733595368988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49660 deadline: 1733595368989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49696 deadline: 1733595368989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49730 deadline: 1733595368998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49676 deadline: 1733595368998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:08,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:08,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:09,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/23533b5e80cc49c29bd40bbffa9308f1 2024-12-07T18:15:09,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/318beef7128a4499af1500af5ebc2765 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/318beef7128a4499af1500af5ebc2765 2024-12-07T18:15:09,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/318beef7128a4499af1500af5ebc2765, entries=150, sequenceid=422, filesize=12.0 K 2024-12-07T18:15:09,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/243b5de1bd904a3ca358929be3f66be7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/243b5de1bd904a3ca358929be3f66be7 2024-12-07T18:15:09,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/243b5de1bd904a3ca358929be3f66be7, entries=150, sequenceid=422, filesize=12.0 K 2024-12-07T18:15:09,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/23533b5e80cc49c29bd40bbffa9308f1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/23533b5e80cc49c29bd40bbffa9308f1 2024-12-07T18:15:09,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/23533b5e80cc49c29bd40bbffa9308f1, entries=150, sequenceid=422, filesize=12.0 K 2024-12-07T18:15:09,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0fc774415149ceeafbb0c8c46caa372c in 1245ms, sequenceid=422, compaction requested=true 2024-12-07T18:15:09,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:09,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:09,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:09,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:09,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:09,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fc774415149ceeafbb0c8c46caa372c:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:09,077 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:09,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:09,077 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:09,078 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:09,078 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:09,078 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/A is initiating minor compaction (all files) 2024-12-07T18:15:09,078 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/B is initiating minor compaction (all files) 2024-12-07T18:15:09,078 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/A in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:09,078 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/B in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:09,078 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4a46928a08c04cd582c1c94ce87cbc4f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/254dfa0c327b4f7bbfd39720d014f352, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/318beef7128a4499af1500af5ebc2765] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.9 K 2024-12-07T18:15:09,078 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a73c78a9774d45f5a92156b91038823f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3f5b569d1aa14fe89af47a549a38abc5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/243b5de1bd904a3ca358929be3f66be7] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.9 K 2024-12-07T18:15:09,078 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a46928a08c04cd582c1c94ce87cbc4f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733595305439 2024-12-07T18:15:09,078 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting a73c78a9774d45f5a92156b91038823f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733595305439 2024-12-07T18:15:09,078 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254dfa0c327b4f7bbfd39720d014f352, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733595305525 2024-12-07T18:15:09,078 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f5b569d1aa14fe89af47a549a38abc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733595305525 2024-12-07T18:15:09,079 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 318beef7128a4499af1500af5ebc2765, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733595306691 2024-12-07T18:15:09,079 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 243b5de1bd904a3ca358929be3f66be7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733595306691 2024-12-07T18:15:09,085 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#A#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:09,085 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#B#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:09,085 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/de8fe767a0dc42d5b847f7052dce3ccc is 50, key is test_row_0/A:col10/1733595306691/Put/seqid=0 2024-12-07T18:15:09,085 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/034db1d5afca4fbab4e311e6e8b3eba4 is 50, key is test_row_0/B:col10/1733595306691/Put/seqid=0 2024-12-07T18:15:09,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742166_1342 (size=13289) 2024-12-07T18:15:09,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742167_1343 (size=13289) 2024-12-07T18:15:09,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-07T18:15:09,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:09,128 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T18:15:09,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:09,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:09,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:09,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:09,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:09,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:09,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/24a8b7be97c3479c8f77cf2b0c23a90b is 50, key is test_row_0/A:col10/1733595307852/Put/seqid=0 2024-12-07T18:15:09,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742168_1344 (size=12301) 2024-12-07T18:15:09,494 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/de8fe767a0dc42d5b847f7052dce3ccc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/de8fe767a0dc42d5b847f7052dce3ccc 2024-12-07T18:15:09,494 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/034db1d5afca4fbab4e311e6e8b3eba4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/034db1d5afca4fbab4e311e6e8b3eba4 2024-12-07T18:15:09,498 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/B of 0fc774415149ceeafbb0c8c46caa372c into 034db1d5afca4fbab4e311e6e8b3eba4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:09,498 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/A of 0fc774415149ceeafbb0c8c46caa372c into de8fe767a0dc42d5b847f7052dce3ccc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:09,498 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:09,498 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:09,498 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/A, priority=13, startTime=1733595309077; duration=0sec 2024-12-07T18:15:09,498 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/B, priority=13, startTime=1733595309077; duration=0sec 2024-12-07T18:15:09,498 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:09,498 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:A 2024-12-07T18:15:09,498 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:09,498 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:B 2024-12-07T18:15:09,498 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:09,499 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:09,499 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fc774415149ceeafbb0c8c46caa372c/C is initiating minor compaction (all files) 2024-12-07T18:15:09,499 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fc774415149ceeafbb0c8c46caa372c/C in TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:09,499 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/28b2de4087d34df8b284f193786a7ecf, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/5b018c03da6d4ab1b6e00be1c2449c3e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/23533b5e80cc49c29bd40bbffa9308f1] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp, totalSize=36.9 K 2024-12-07T18:15:09,500 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28b2de4087d34df8b284f193786a7ecf, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1733595305439 2024-12-07T18:15:09,500 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b018c03da6d4ab1b6e00be1c2449c3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733595305525 2024-12-07T18:15:09,500 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23533b5e80cc49c29bd40bbffa9308f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1733595306691 2024-12-07T18:15:09,505 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fc774415149ceeafbb0c8c46caa372c#C#compaction#297 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:09,506 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/8d38eb772ba74ee28760fa96f2d5c79b is 50, key is test_row_0/C:col10/1733595306691/Put/seqid=0 2024-12-07T18:15:09,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742169_1345 (size=13289) 2024-12-07T18:15:09,535 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/24a8b7be97c3479c8f77cf2b0c23a90b 2024-12-07T18:15:09,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/0542d95ce4604980b30bdd4917879af7 is 50, key is test_row_0/B:col10/1733595307852/Put/seqid=0 2024-12-07T18:15:09,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742170_1346 (size=12301) 2024-12-07T18:15:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T18:15:09,914 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/8d38eb772ba74ee28760fa96f2d5c79b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/8d38eb772ba74ee28760fa96f2d5c79b 2024-12-07T18:15:09,919 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fc774415149ceeafbb0c8c46caa372c/C of 0fc774415149ceeafbb0c8c46caa372c into 8d38eb772ba74ee28760fa96f2d5c79b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:09,919 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:09,919 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., storeName=0fc774415149ceeafbb0c8c46caa372c/C, priority=13, startTime=1733595309077; duration=0sec 2024-12-07T18:15:09,919 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:09,919 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fc774415149ceeafbb0c8c46caa372c:C 2024-12-07T18:15:09,946 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/0542d95ce4604980b30bdd4917879af7 2024-12-07T18:15:09,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/638935b6243e42018efdf8d5dcc79748 is 50, key is test_row_0/C:col10/1733595307852/Put/seqid=0 2024-12-07T18:15:09,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742171_1347 (size=12301) 2024-12-07T18:15:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:09,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. as already flushing 2024-12-07T18:15:09,999 INFO [Thread-1178 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1178. 2024-12-07T18:15:09,999 DEBUG [Thread-1178 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:09,999 DEBUG [Thread-1178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:09,999 DEBUG [Thread-1178 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:09,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:10,010 INFO [Thread-1184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1184. 2024-12-07T18:15:10,010 INFO [Thread-1180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1180. 2024-12-07T18:15:10,010 INFO [Thread-1186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1186. 2024-12-07T18:15:10,010 DEBUG [Thread-1184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:10,010 DEBUG [Thread-1180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:10,010 DEBUG [Thread-1186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:10,010 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:10,010 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:10,010 DEBUG [Thread-1180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:10,010 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:10,010 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:10,010 DEBUG [Thread-1180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:10,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:10,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:10,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:10,011 INFO [Thread-1182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1182. 2024-12-07T18:15:10,011 DEBUG [Thread-1182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:10,011 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:10,011 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:10,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:10,356 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/638935b6243e42018efdf8d5dcc79748 2024-12-07T18:15:10,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/24a8b7be97c3479c8f77cf2b0c23a90b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/24a8b7be97c3479c8f77cf2b0c23a90b 2024-12-07T18:15:10,364 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/24a8b7be97c3479c8f77cf2b0c23a90b, entries=150, sequenceid=440, filesize=12.0 K 2024-12-07T18:15:10,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/0542d95ce4604980b30bdd4917879af7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/0542d95ce4604980b30bdd4917879af7 2024-12-07T18:15:10,367 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/0542d95ce4604980b30bdd4917879af7, entries=150, sequenceid=440, filesize=12.0 K 2024-12-07T18:15:10,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/638935b6243e42018efdf8d5dcc79748 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/638935b6243e42018efdf8d5dcc79748 2024-12-07T18:15:10,370 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/638935b6243e42018efdf8d5dcc79748, entries=150, sequenceid=440, filesize=12.0 K 2024-12-07T18:15:10,371 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=33.54 KB/34350 for 0fc774415149ceeafbb0c8c46caa372c in 1243ms, sequenceid=440, compaction requested=false 2024-12-07T18:15:10,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 0fc774415149ceeafbb0c8c46caa372c: 2024-12-07T18:15:10,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:10,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-07T18:15:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-07T18:15:10,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-07T18:15:10,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6200 sec 2024-12-07T18:15:10,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.6250 sec 2024-12-07T18:15:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-07T18:15:11,919 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3058 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3017 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3034 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3078 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3035 2024-12-07T18:15:11,919 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T18:15:11,919 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:15:11,919 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:15:11,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:11,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:11,920 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:11,920 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T18:15:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:11,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T18:15:11,923 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595311923"}]},"ts":"1733595311923"} 2024-12-07T18:15:11,924 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T18:15:11,924 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T18:15:11,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T18:15:11,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc774415149ceeafbb0c8c46caa372c, UNASSIGN}] 2024-12-07T18:15:11,927 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc774415149ceeafbb0c8c46caa372c, UNASSIGN 2024-12-07T18:15:11,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=0fc774415149ceeafbb0c8c46caa372c, regionState=CLOSING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:11,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc774415149ceeafbb0c8c46caa372c, UNASSIGN because future has completed 2024-12-07T18:15:11,928 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T18:15:11,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:15:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T18:15:12,080 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 0fc774415149ceeafbb0c8c46caa372c, disabling compactions & flushes 2024-12-07T18:15:12,081 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. after waiting 0 ms 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:12,081 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 0fc774415149ceeafbb0c8c46caa372c 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=A 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=B 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fc774415149ceeafbb0c8c46caa372c, store=C 2024-12-07T18:15:12,081 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:12,085 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/a1a21bfffa404d5fbde666c35e3f8ee4 is 50, key is test_row_1/A:col10/1733595310010/Put/seqid=0 2024-12-07T18:15:12,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742172_1348 (size=9857) 2024-12-07T18:15:12,089 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/a1a21bfffa404d5fbde666c35e3f8ee4 2024-12-07T18:15:12,095 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/7a3bc7137b4444e5b9df2040d533d137 is 50, key is test_row_1/B:col10/1733595310010/Put/seqid=0 2024-12-07T18:15:12,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742173_1349 (size=9857) 2024-12-07T18:15:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T18:15:12,499 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/7a3bc7137b4444e5b9df2040d533d137 2024-12-07T18:15:12,505 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/23ea4fd505f541d0977b78c6c7171ab8 is 50, key is test_row_1/C:col10/1733595310010/Put/seqid=0 2024-12-07T18:15:12,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742174_1350 (size=9857) 2024-12-07T18:15:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T18:15:12,909 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/23ea4fd505f541d0977b78c6c7171ab8 2024-12-07T18:15:12,913 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/A/a1a21bfffa404d5fbde666c35e3f8ee4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/a1a21bfffa404d5fbde666c35e3f8ee4 2024-12-07T18:15:12,916 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/a1a21bfffa404d5fbde666c35e3f8ee4, entries=100, sequenceid=451, filesize=9.6 K 2024-12-07T18:15:12,917 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/B/7a3bc7137b4444e5b9df2040d533d137 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/7a3bc7137b4444e5b9df2040d533d137 2024-12-07T18:15:12,920 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/7a3bc7137b4444e5b9df2040d533d137, entries=100, sequenceid=451, filesize=9.6 K 2024-12-07T18:15:12,921 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/.tmp/C/23ea4fd505f541d0977b78c6c7171ab8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/23ea4fd505f541d0977b78c6c7171ab8 2024-12-07T18:15:12,924 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/23ea4fd505f541d0977b78c6c7171ab8, entries=100, sequenceid=451, filesize=9.6 K 2024-12-07T18:15:12,924 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0fc774415149ceeafbb0c8c46caa372c in 843ms, sequenceid=451, compaction requested=true 2024-12-07T18:15:12,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/7c0626dcc9814e488e875a4151737188, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/a141e9a799d843fcad10c64d495bdff2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/49970c60f8204fc2849ec71619ffe554, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/74b9eafaddb9424c8281704487d6e462, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/43bdf4a41d7342c7a80750ec52bea0ee, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d56e4a1348dc4ad7aa00ffca91c2bc69, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/052bd429fe0f4ad3933adab6da5f5d63, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/e5a63458efaf499387b302b2bb85ec88, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/1111b569073f48cbadd717e0ff5973d1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/722a4fdcccd34be9b40d93909b160bdc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/449d1414dfa540d08d327244ddfed2fc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/eba085d67d75440e861df4a4a2e4428a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d7ee68fd188649599e0147e2d5169c2e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/44763859541f47258987b5ad4501ea05, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/f8c0b2ffb4424bb59a5a863c361d3b6d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/6bc5f72ff0714f009aac33f06f8777a1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/c7e91fbfd0c14ec68e615e9662e28757, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3472330dab964cfd97e09df8e04e1b69, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3d9ad2aeabec442892421e18f296aa9f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/5f29bbc174ab4a0dacbd376914693598, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/843aec477b1642ba9d7806c505069aae, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69456a031889430880e3e5665712b891, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69335f8d9b3441319ad0580668732423, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/9bd8a1804b654cc19dc158d81cdd1865, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4997235d7bf4441b9173fafa76395595, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4a46928a08c04cd582c1c94ce87cbc4f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/b04f09bfdc1b47a19c887ea92c53b810, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/254dfa0c327b4f7bbfd39720d014f352, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/318beef7128a4499af1500af5ebc2765] to archive 2024-12-07T18:15:12,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:15:12,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/7c0626dcc9814e488e875a4151737188 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/7c0626dcc9814e488e875a4151737188 2024-12-07T18:15:12,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/a141e9a799d843fcad10c64d495bdff2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/a141e9a799d843fcad10c64d495bdff2 2024-12-07T18:15:12,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/49970c60f8204fc2849ec71619ffe554 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/49970c60f8204fc2849ec71619ffe554 2024-12-07T18:15:12,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/74b9eafaddb9424c8281704487d6e462 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/74b9eafaddb9424c8281704487d6e462 2024-12-07T18:15:12,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/43bdf4a41d7342c7a80750ec52bea0ee to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/43bdf4a41d7342c7a80750ec52bea0ee 2024-12-07T18:15:12,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d56e4a1348dc4ad7aa00ffca91c2bc69 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d56e4a1348dc4ad7aa00ffca91c2bc69 2024-12-07T18:15:12,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/052bd429fe0f4ad3933adab6da5f5d63 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/052bd429fe0f4ad3933adab6da5f5d63 2024-12-07T18:15:12,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/e5a63458efaf499387b302b2bb85ec88 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/e5a63458efaf499387b302b2bb85ec88 2024-12-07T18:15:12,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/1111b569073f48cbadd717e0ff5973d1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/1111b569073f48cbadd717e0ff5973d1 2024-12-07T18:15:12,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/722a4fdcccd34be9b40d93909b160bdc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/722a4fdcccd34be9b40d93909b160bdc 2024-12-07T18:15:12,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/449d1414dfa540d08d327244ddfed2fc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/449d1414dfa540d08d327244ddfed2fc 2024-12-07T18:15:12,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/eba085d67d75440e861df4a4a2e4428a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/eba085d67d75440e861df4a4a2e4428a 2024-12-07T18:15:12,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d7ee68fd188649599e0147e2d5169c2e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/d7ee68fd188649599e0147e2d5169c2e 2024-12-07T18:15:12,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/44763859541f47258987b5ad4501ea05 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/44763859541f47258987b5ad4501ea05 2024-12-07T18:15:12,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/f8c0b2ffb4424bb59a5a863c361d3b6d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/f8c0b2ffb4424bb59a5a863c361d3b6d 2024-12-07T18:15:12,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/6bc5f72ff0714f009aac33f06f8777a1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/6bc5f72ff0714f009aac33f06f8777a1 2024-12-07T18:15:12,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/c7e91fbfd0c14ec68e615e9662e28757 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/c7e91fbfd0c14ec68e615e9662e28757 2024-12-07T18:15:12,942 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3472330dab964cfd97e09df8e04e1b69 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3472330dab964cfd97e09df8e04e1b69 2024-12-07T18:15:12,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3d9ad2aeabec442892421e18f296aa9f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/3d9ad2aeabec442892421e18f296aa9f 2024-12-07T18:15:12,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/5f29bbc174ab4a0dacbd376914693598 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/5f29bbc174ab4a0dacbd376914693598 2024-12-07T18:15:12,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/843aec477b1642ba9d7806c505069aae to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/843aec477b1642ba9d7806c505069aae 2024-12-07T18:15:12,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69456a031889430880e3e5665712b891 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69456a031889430880e3e5665712b891 2024-12-07T18:15:12,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69335f8d9b3441319ad0580668732423 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/69335f8d9b3441319ad0580668732423 2024-12-07T18:15:12,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/9bd8a1804b654cc19dc158d81cdd1865 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/9bd8a1804b654cc19dc158d81cdd1865 2024-12-07T18:15:12,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4997235d7bf4441b9173fafa76395595 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4997235d7bf4441b9173fafa76395595 2024-12-07T18:15:12,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4a46928a08c04cd582c1c94ce87cbc4f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/4a46928a08c04cd582c1c94ce87cbc4f 2024-12-07T18:15:12,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/b04f09bfdc1b47a19c887ea92c53b810 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/b04f09bfdc1b47a19c887ea92c53b810 2024-12-07T18:15:12,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/254dfa0c327b4f7bbfd39720d014f352 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/254dfa0c327b4f7bbfd39720d014f352 2024-12-07T18:15:12,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/318beef7128a4499af1500af5ebc2765 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/318beef7128a4499af1500af5ebc2765 2024-12-07T18:15:12,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c151adbffc842488296743c6f72d95c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/1600a1bafaaa41bcbfc2a9b2bf73fada, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/dd356eab006b49028cfc0b60cc68e4fd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/c22433f9c9084cf38fda01d1883f9d4c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fc3242fa12344ca8a35b9dcbb34acde2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/377e859ba3f648dd8accc7d4db7afdf6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a913add0d0db4918997cc5a29a69b84a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c9e7a6372b1413980b87ff30d03d0fd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/939555ff6c344ba1a6475e791c74bc39, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3fc839d118414490b0963e2c8b19821d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6b5dd338a84847cb894fab57ff907fb2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fd0a576953f846fa81d115e327a4be8e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/228d2e0f1b8d43a9b0a73c908444a270, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/0e363836899340fb925c34c01fc9774b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/94ee575c08d24f51b90aa83d1bda285f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/26dac45a61b54c76ab9bd4128abfb2c2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/9692467403414cdd84cca744a35ed207, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e5af8b34e28146e89e9d240f9fa8616f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6f37daa98d04424c9a7f73b3b23c126c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6cb1a1a08e65463eb695f2d4b56f3af9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/634a17a3683f4246b9f8c08ecb8f5d04, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/ab91ce678f414079ac6d46f110baedf2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/9cf6da06eb49488ea0cd180d123539b7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e9b5951821724f85b437197ad5c517ca, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/69120230782c448dad2fa112e3dd1fb0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a73c78a9774d45f5a92156b91038823f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/f4d6e02d6ec348279a6c7e05960ed8db, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3f5b569d1aa14fe89af47a549a38abc5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/243b5de1bd904a3ca358929be3f66be7] to archive 2024-12-07T18:15:12,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:15:12,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c151adbffc842488296743c6f72d95c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c151adbffc842488296743c6f72d95c 2024-12-07T18:15:12,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/1600a1bafaaa41bcbfc2a9b2bf73fada to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/1600a1bafaaa41bcbfc2a9b2bf73fada 2024-12-07T18:15:12,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/dd356eab006b49028cfc0b60cc68e4fd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/dd356eab006b49028cfc0b60cc68e4fd 2024-12-07T18:15:12,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/c22433f9c9084cf38fda01d1883f9d4c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/c22433f9c9084cf38fda01d1883f9d4c 2024-12-07T18:15:12,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fc3242fa12344ca8a35b9dcbb34acde2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fc3242fa12344ca8a35b9dcbb34acde2 2024-12-07T18:15:12,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/377e859ba3f648dd8accc7d4db7afdf6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/377e859ba3f648dd8accc7d4db7afdf6 2024-12-07T18:15:12,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a913add0d0db4918997cc5a29a69b84a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a913add0d0db4918997cc5a29a69b84a 2024-12-07T18:15:12,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c9e7a6372b1413980b87ff30d03d0fd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/2c9e7a6372b1413980b87ff30d03d0fd 2024-12-07T18:15:12,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/939555ff6c344ba1a6475e791c74bc39 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/939555ff6c344ba1a6475e791c74bc39 2024-12-07T18:15:12,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3fc839d118414490b0963e2c8b19821d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3fc839d118414490b0963e2c8b19821d 2024-12-07T18:15:12,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6b5dd338a84847cb894fab57ff907fb2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6b5dd338a84847cb894fab57ff907fb2 2024-12-07T18:15:12,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fd0a576953f846fa81d115e327a4be8e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/fd0a576953f846fa81d115e327a4be8e 2024-12-07T18:15:12,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/228d2e0f1b8d43a9b0a73c908444a270 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/228d2e0f1b8d43a9b0a73c908444a270 2024-12-07T18:15:12,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/0e363836899340fb925c34c01fc9774b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/0e363836899340fb925c34c01fc9774b 2024-12-07T18:15:12,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/94ee575c08d24f51b90aa83d1bda285f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/94ee575c08d24f51b90aa83d1bda285f 2024-12-07T18:15:12,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/26dac45a61b54c76ab9bd4128abfb2c2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/26dac45a61b54c76ab9bd4128abfb2c2 2024-12-07T18:15:12,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/9692467403414cdd84cca744a35ed207 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/9692467403414cdd84cca744a35ed207 2024-12-07T18:15:12,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e5af8b34e28146e89e9d240f9fa8616f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e5af8b34e28146e89e9d240f9fa8616f 2024-12-07T18:15:12,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6f37daa98d04424c9a7f73b3b23c126c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6f37daa98d04424c9a7f73b3b23c126c 2024-12-07T18:15:12,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6cb1a1a08e65463eb695f2d4b56f3af9 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/6cb1a1a08e65463eb695f2d4b56f3af9 2024-12-07T18:15:12,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/634a17a3683f4246b9f8c08ecb8f5d04 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/634a17a3683f4246b9f8c08ecb8f5d04 2024-12-07T18:15:12,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/ab91ce678f414079ac6d46f110baedf2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/ab91ce678f414079ac6d46f110baedf2 2024-12-07T18:15:12,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/9cf6da06eb49488ea0cd180d123539b7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/9cf6da06eb49488ea0cd180d123539b7 2024-12-07T18:15:12,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e9b5951821724f85b437197ad5c517ca to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/e9b5951821724f85b437197ad5c517ca 2024-12-07T18:15:12,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/69120230782c448dad2fa112e3dd1fb0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/69120230782c448dad2fa112e3dd1fb0 2024-12-07T18:15:12,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a73c78a9774d45f5a92156b91038823f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/a73c78a9774d45f5a92156b91038823f 2024-12-07T18:15:12,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/f4d6e02d6ec348279a6c7e05960ed8db to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/f4d6e02d6ec348279a6c7e05960ed8db 2024-12-07T18:15:12,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3f5b569d1aa14fe89af47a549a38abc5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/3f5b569d1aa14fe89af47a549a38abc5 2024-12-07T18:15:12,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/243b5de1bd904a3ca358929be3f66be7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/243b5de1bd904a3ca358929be3f66be7 2024-12-07T18:15:12,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea76d6de0569425189a075ccaf825396, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/370d710984b34d439295d27f8f492102, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/16542af5974643b5b57f8e0bdd4b1738, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c2ab959650e44be0a05d36bb2b3a9608, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/8e228c2d411144488e61f88a69ae9653, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/d0e2dedb33c24c1eaac55f1cdff8bbf6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/58cf3ad9b52f48e6ae252d63e6997f51, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c5ee8b6298464fbf8135ece5aa5e9deb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/7f03a1a7e2634e228299976c89448743, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/287a70043c694b54a2807d8f6ac5617b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/e7c638032f0f40c1b6d49d653ea64f50, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/bf448a6383804fa9853db0f417bf8e1f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/04b8cb2dc1824c4ebccdedd9c0156fbd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/339088a84eac4f0aa502653b4c2f4cc5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/fe8db5886f39468ea047ea06b65f44e2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/6e623e9a38f648219fd590f70a2dbf56, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ef842b7aa99748a3a46029787db77b04, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/54f717cefd0f4098a5427e991ab1a814, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4452f5ce14124efc986226474e652727, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/64a0dd9eb9114e03ac4536964fd0a932, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4c989033878840d499d286ae8accd201, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/2433079dd0b94a6dbe4b1e9a4a558a23, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/22b96980aad34dac92972103ff9b2afe, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c1b3551ce5e34bddbb3e0be1ff177bf4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/068f1d09d8f14f85b711dc8a422cdf6a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/28b2de4087d34df8b284f193786a7ecf, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea4ae9b36efc4039897db6885159b1c1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/5b018c03da6d4ab1b6e00be1c2449c3e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/23533b5e80cc49c29bd40bbffa9308f1] to archive 2024-12-07T18:15:12,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:15:12,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea76d6de0569425189a075ccaf825396 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea76d6de0569425189a075ccaf825396 2024-12-07T18:15:12,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/370d710984b34d439295d27f8f492102 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/370d710984b34d439295d27f8f492102 2024-12-07T18:15:12,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/16542af5974643b5b57f8e0bdd4b1738 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/16542af5974643b5b57f8e0bdd4b1738 2024-12-07T18:15:12,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c2ab959650e44be0a05d36bb2b3a9608 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c2ab959650e44be0a05d36bb2b3a9608 2024-12-07T18:15:12,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/8e228c2d411144488e61f88a69ae9653 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/8e228c2d411144488e61f88a69ae9653 2024-12-07T18:15:12,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/d0e2dedb33c24c1eaac55f1cdff8bbf6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/d0e2dedb33c24c1eaac55f1cdff8bbf6 2024-12-07T18:15:12,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/58cf3ad9b52f48e6ae252d63e6997f51 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/58cf3ad9b52f48e6ae252d63e6997f51 2024-12-07T18:15:12,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c5ee8b6298464fbf8135ece5aa5e9deb to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c5ee8b6298464fbf8135ece5aa5e9deb 2024-12-07T18:15:12,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/7f03a1a7e2634e228299976c89448743 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/7f03a1a7e2634e228299976c89448743 2024-12-07T18:15:12,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/287a70043c694b54a2807d8f6ac5617b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/287a70043c694b54a2807d8f6ac5617b 2024-12-07T18:15:13,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/e7c638032f0f40c1b6d49d653ea64f50 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/e7c638032f0f40c1b6d49d653ea64f50 2024-12-07T18:15:13,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/bf448a6383804fa9853db0f417bf8e1f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/bf448a6383804fa9853db0f417bf8e1f 2024-12-07T18:15:13,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/04b8cb2dc1824c4ebccdedd9c0156fbd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/04b8cb2dc1824c4ebccdedd9c0156fbd 2024-12-07T18:15:13,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/339088a84eac4f0aa502653b4c2f4cc5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/339088a84eac4f0aa502653b4c2f4cc5 2024-12-07T18:15:13,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/fe8db5886f39468ea047ea06b65f44e2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/fe8db5886f39468ea047ea06b65f44e2 2024-12-07T18:15:13,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/6e623e9a38f648219fd590f70a2dbf56 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/6e623e9a38f648219fd590f70a2dbf56 2024-12-07T18:15:13,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ef842b7aa99748a3a46029787db77b04 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ef842b7aa99748a3a46029787db77b04 2024-12-07T18:15:13,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/54f717cefd0f4098a5427e991ab1a814 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/54f717cefd0f4098a5427e991ab1a814 2024-12-07T18:15:13,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4452f5ce14124efc986226474e652727 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4452f5ce14124efc986226474e652727 2024-12-07T18:15:13,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/64a0dd9eb9114e03ac4536964fd0a932 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/64a0dd9eb9114e03ac4536964fd0a932 2024-12-07T18:15:13,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4c989033878840d499d286ae8accd201 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/4c989033878840d499d286ae8accd201 2024-12-07T18:15:13,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/2433079dd0b94a6dbe4b1e9a4a558a23 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/2433079dd0b94a6dbe4b1e9a4a558a23 2024-12-07T18:15:13,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/22b96980aad34dac92972103ff9b2afe to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/22b96980aad34dac92972103ff9b2afe 2024-12-07T18:15:13,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c1b3551ce5e34bddbb3e0be1ff177bf4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/c1b3551ce5e34bddbb3e0be1ff177bf4 2024-12-07T18:15:13,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/068f1d09d8f14f85b711dc8a422cdf6a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/068f1d09d8f14f85b711dc8a422cdf6a 2024-12-07T18:15:13,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/28b2de4087d34df8b284f193786a7ecf to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/28b2de4087d34df8b284f193786a7ecf 2024-12-07T18:15:13,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea4ae9b36efc4039897db6885159b1c1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/ea4ae9b36efc4039897db6885159b1c1 2024-12-07T18:15:13,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/5b018c03da6d4ab1b6e00be1c2449c3e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/5b018c03da6d4ab1b6e00be1c2449c3e 2024-12-07T18:15:13,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/23533b5e80cc49c29bd40bbffa9308f1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/23533b5e80cc49c29bd40bbffa9308f1 2024-12-07T18:15:13,022 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/recovered.edits/454.seqid, newMaxSeqId=454, maxSeqId=1 2024-12-07T18:15:13,023 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. 2024-12-07T18:15:13,023 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 0fc774415149ceeafbb0c8c46caa372c: Waiting for close lock at 1733595312081Running coprocessor pre-close hooks at 1733595312081Disabling compacts and flushes for region at 1733595312081Disabling writes for close at 1733595312081Obtaining lock to block concurrent updates at 1733595312081Preparing flush snapshotting stores in 0fc774415149ceeafbb0c8c46caa372c at 1733595312081Finished memstore snapshotting TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733595312081Flushing stores of TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c. at 1733595312082 (+1 ms)Flushing 0fc774415149ceeafbb0c8c46caa372c/A: creating writer at 1733595312082Flushing 0fc774415149ceeafbb0c8c46caa372c/A: appending metadata at 1733595312084 (+2 ms)Flushing 0fc774415149ceeafbb0c8c46caa372c/A: closing flushed file at 1733595312084Flushing 0fc774415149ceeafbb0c8c46caa372c/B: creating writer at 1733595312093 (+9 ms)Flushing 0fc774415149ceeafbb0c8c46caa372c/B: appending metadata at 1733595312095 (+2 ms)Flushing 0fc774415149ceeafbb0c8c46caa372c/B: closing flushed file at 1733595312095Flushing 0fc774415149ceeafbb0c8c46caa372c/C: creating writer at 1733595312502 (+407 ms)Flushing 0fc774415149ceeafbb0c8c46caa372c/C: appending metadata at 1733595312505 (+3 ms)Flushing 0fc774415149ceeafbb0c8c46caa372c/C: closing flushed file at 1733595312505Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a62baec: reopening flushed file at 1733595312912 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55c87863: reopening flushed file at 1733595312916 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2abdce66: reopening flushed file at 1733595312920 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0fc774415149ceeafbb0c8c46caa372c in 843ms, sequenceid=451, compaction requested=true at 1733595312924 (+4 ms)Writing region close event to WAL at 1733595313019 (+95 ms)Running coprocessor post-close hooks at 1733595313023 (+4 ms)Closed at 1733595313023 2024-12-07T18:15:13,025 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:13,026 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=0fc774415149ceeafbb0c8c46caa372c, regionState=CLOSED 2024-12-07T18:15:13,027 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:15:13,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-07T18:15:13,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 0fc774415149ceeafbb0c8c46caa372c, server=92acd7e2389b,33947,1733595217075 in 1.1000 sec 2024-12-07T18:15:13,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-07T18:15:13,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fc774415149ceeafbb0c8c46caa372c, UNASSIGN in 1.1030 sec 2024-12-07T18:15:13,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-07T18:15:13,033 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1060 sec 2024-12-07T18:15:13,034 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595313034"}]},"ts":"1733595313034"} 2024-12-07T18:15:13,035 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T18:15:13,035 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T18:15:13,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1150 sec 2024-12-07T18:15:13,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-07T18:15:13,059 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:13,059 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T18:15:13,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:13,061 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:13,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-07T18:15:13,062 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:13,063 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:13,065 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/recovered.edits] 2024-12-07T18:15:13,068 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/24a8b7be97c3479c8f77cf2b0c23a90b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/24a8b7be97c3479c8f77cf2b0c23a90b 2024-12-07T18:15:13,069 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/a1a21bfffa404d5fbde666c35e3f8ee4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/a1a21bfffa404d5fbde666c35e3f8ee4 2024-12-07T18:15:13,070 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/de8fe767a0dc42d5b847f7052dce3ccc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/A/de8fe767a0dc42d5b847f7052dce3ccc 2024-12-07T18:15:13,072 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/034db1d5afca4fbab4e311e6e8b3eba4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/034db1d5afca4fbab4e311e6e8b3eba4 2024-12-07T18:15:13,073 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/0542d95ce4604980b30bdd4917879af7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/0542d95ce4604980b30bdd4917879af7 2024-12-07T18:15:13,074 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/7a3bc7137b4444e5b9df2040d533d137 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/B/7a3bc7137b4444e5b9df2040d533d137 2024-12-07T18:15:13,076 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/23ea4fd505f541d0977b78c6c7171ab8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/23ea4fd505f541d0977b78c6c7171ab8 2024-12-07T18:15:13,077 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/638935b6243e42018efdf8d5dcc79748 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/638935b6243e42018efdf8d5dcc79748 2024-12-07T18:15:13,078 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/8d38eb772ba74ee28760fa96f2d5c79b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/C/8d38eb772ba74ee28760fa96f2d5c79b 2024-12-07T18:15:13,080 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/recovered.edits/454.seqid to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c/recovered.edits/454.seqid 2024-12-07T18:15:13,081 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fc774415149ceeafbb0c8c46caa372c 2024-12-07T18:15:13,081 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T18:15:13,082 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:13,085 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T18:15:13,086 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T18:15:13,087 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:13,087 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T18:15:13,088 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733595313087"}]},"ts":"9223372036854775807"} 2024-12-07T18:15:13,089 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T18:15:13,089 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0fc774415149ceeafbb0c8c46caa372c, NAME => 'TestAcidGuarantees,,1733595285881.0fc774415149ceeafbb0c8c46caa372c.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T18:15:13,089 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T18:15:13,089 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733595313089"}]},"ts":"9223372036854775807"} 2024-12-07T18:15:13,091 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T18:15:13,091 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:13,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-12-07T18:15:13,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-07T18:15:13,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T18:15:13,169 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:13,178 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-17801953_22 at /127.0.0.1:51748 [Waiting for operation #362] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-823063722_22 at /127.0.0.1:42702 [Waiting for operation #232] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=490 (was 414) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6542 (was 6640) 2024-12-07T18:15:13,186 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=490, ProcessCount=11, AvailableMemoryMB=6542 2024-12-07T18:15:13,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T18:15:13,188 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:15:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:13,189 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T18:15:13,189 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:13,189 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-12-07T18:15:13,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-07T18:15:13,190 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T18:15:13,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742175_1351 (size=1061) 2024-12-07T18:15:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-07T18:15:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-07T18:15:13,597 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68 2024-12-07T18:15:13,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742176_1352 (size=53) 2024-12-07T18:15:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-07T18:15:14,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:15:14,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6f3992ce41b26fed1c9fca93d87e0364, disabling compactions & flushes 2024-12-07T18:15:14,002 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:14,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:14,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. after waiting 0 ms 2024-12-07T18:15:14,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:14,002 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:14,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6f3992ce41b26fed1c9fca93d87e0364: Waiting for close lock at 1733595314002Disabling compacts and flushes for region at 1733595314002Disabling writes for close at 1733595314002Writing region close event to WAL at 1733595314002Closed at 1733595314002 2024-12-07T18:15:14,003 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T18:15:14,004 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733595314003"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733595314003"}]},"ts":"1733595314003"} 2024-12-07T18:15:14,005 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T18:15:14,006 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T18:15:14,006 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595314006"}]},"ts":"1733595314006"} 2024-12-07T18:15:14,007 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T18:15:14,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, ASSIGN}] 2024-12-07T18:15:14,008 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, ASSIGN 2024-12-07T18:15:14,009 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, ASSIGN; state=OFFLINE, location=92acd7e2389b,33947,1733595217075; forceNewPlan=false, retain=false 2024-12-07T18:15:14,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=6f3992ce41b26fed1c9fca93d87e0364, regionState=OPENING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:14,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, ASSIGN because future has completed 2024-12-07T18:15:14,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:15:14,315 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:14,316 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:15:14,316 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,316 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:15:14,316 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,316 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,317 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,318 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:15:14,318 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f3992ce41b26fed1c9fca93d87e0364 columnFamilyName A 2024-12-07T18:15:14,319 DEBUG [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:14,319 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(327): Store=6f3992ce41b26fed1c9fca93d87e0364/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:15:14,319 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,320 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:15:14,320 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f3992ce41b26fed1c9fca93d87e0364 columnFamilyName B 2024-12-07T18:15:14,320 DEBUG [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:14,320 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(327): Store=6f3992ce41b26fed1c9fca93d87e0364/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:15:14,320 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,321 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:15:14,321 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f3992ce41b26fed1c9fca93d87e0364 columnFamilyName C 2024-12-07T18:15:14,321 DEBUG [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:14,322 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(327): Store=6f3992ce41b26fed1c9fca93d87e0364/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:15:14,322 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:14,322 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,322 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,323 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,323 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,323 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,324 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:15:14,324 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,326 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:15:14,326 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 6f3992ce41b26fed1c9fca93d87e0364; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67226813, jitterRate=0.0017575770616531372}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:15:14,326 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:14,327 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 6f3992ce41b26fed1c9fca93d87e0364: Running coprocessor pre-open hook at 1733595314316Writing region info on filesystem at 1733595314316Initializing all the Stores at 1733595314317 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595314317Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595314317Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595314317Cleaning up temporary data from old regions at 1733595314323 (+6 ms)Running coprocessor post-open hooks at 1733595314326 (+3 ms)Region opened successfully at 1733595314327 (+1 ms) 2024-12-07T18:15:14,328 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., pid=91, masterSystemTime=1733595314313 2024-12-07T18:15:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-07T18:15:14,329 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:14,329 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:14,329 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=6f3992ce41b26fed1c9fca93d87e0364, regionState=OPEN, openSeqNum=2, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:14,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:15:14,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-07T18:15:14,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 in 170 msec 2024-12-07T18:15:14,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-07T18:15:14,334 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, ASSIGN in 325 msec 2024-12-07T18:15:14,334 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T18:15:14,334 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595314334"}]},"ts":"1733595314334"} 2024-12-07T18:15:14,335 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T18:15:14,336 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T18:15:14,337 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-12-07T18:15:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-07T18:15:15,338 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:15,339 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300a5f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:15,339 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:15,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:15,340 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4864b342, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:15,342 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38dc69a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:15,343 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:15,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:15,345 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T18:15:15,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 92acd7e2389b,38531,1733595216342 2024-12-07T18:15:15,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2edffdf5 2024-12-07T18:15:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T18:15:15,346 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T18:15:15,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T18:15:15,348 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:15:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:15,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742177_1353 (size=1097) 2024-12-07T18:15:15,759 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-07T18:15:15,759 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-07T18:15:15,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T18:15:15,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, REOPEN/MOVE}] 2024-12-07T18:15:15,763 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, REOPEN/MOVE 2024-12-07T18:15:15,764 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6f3992ce41b26fed1c9fca93d87e0364, regionState=CLOSING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:15,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, REOPEN/MOVE because future has completed 2024-12-07T18:15:15,765 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T18:15:15,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:15:15,917 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:15,917 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T18:15:15,917 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 6f3992ce41b26fed1c9fca93d87e0364, disabling compactions & flushes 2024-12-07T18:15:15,917 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:15,917 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:15,917 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. after waiting 0 ms 2024-12-07T18:15:15,918 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:15,921 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T18:15:15,921 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:15,921 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 6f3992ce41b26fed1c9fca93d87e0364: Waiting for close lock at 1733595315917Running coprocessor pre-close hooks at 1733595315917Disabling compacts and flushes for region at 1733595315917Disabling writes for close at 1733595315918 (+1 ms)Writing region close event to WAL at 1733595315918Running coprocessor post-close hooks at 1733595315921 (+3 ms)Closed at 1733595315921 2024-12-07T18:15:15,921 WARN [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 6f3992ce41b26fed1c9fca93d87e0364 to self. 2024-12-07T18:15:15,923 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:15,923 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6f3992ce41b26fed1c9fca93d87e0364, regionState=CLOSED 2024-12-07T18:15:15,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:15:15,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-07T18:15:15,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 in 160 msec 2024-12-07T18:15:15,927 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, REOPEN/MOVE; state=CLOSED, location=92acd7e2389b,33947,1733595217075; forceNewPlan=false, retain=true 2024-12-07T18:15:16,077 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6f3992ce41b26fed1c9fca93d87e0364, regionState=OPENING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, REOPEN/MOVE because future has completed 2024-12-07T18:15:16,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:15:16,233 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,234 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:15:16,234 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,234 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:15:16,234 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,234 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,235 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,236 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:15:16,236 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f3992ce41b26fed1c9fca93d87e0364 columnFamilyName A 2024-12-07T18:15:16,237 DEBUG [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:16,237 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(327): Store=6f3992ce41b26fed1c9fca93d87e0364/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:15:16,238 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,238 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:15:16,238 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f3992ce41b26fed1c9fca93d87e0364 columnFamilyName B 2024-12-07T18:15:16,238 DEBUG [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:16,239 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(327): Store=6f3992ce41b26fed1c9fca93d87e0364/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:15:16,239 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,239 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:15:16,239 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f3992ce41b26fed1c9fca93d87e0364 columnFamilyName C 2024-12-07T18:15:16,239 DEBUG [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:16,240 INFO [StoreOpener-6f3992ce41b26fed1c9fca93d87e0364-1 {}] regionserver.HStore(327): Store=6f3992ce41b26fed1c9fca93d87e0364/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:15:16,240 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,240 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,241 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,242 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,242 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,242 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,243 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:15:16,244 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,244 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 6f3992ce41b26fed1c9fca93d87e0364; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63130059, jitterRate=-0.059288814663887024}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:15:16,244 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,245 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 6f3992ce41b26fed1c9fca93d87e0364: Running coprocessor pre-open hook at 1733595316234Writing region info on filesystem at 1733595316234Initializing all the Stores at 1733595316235 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595316235Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595316235Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595316235Cleaning up temporary data from old regions at 1733595316242 (+7 ms)Running coprocessor post-open hooks at 1733595316244 (+2 ms)Region opened successfully at 1733595316245 (+1 ms) 2024-12-07T18:15:16,246 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., pid=96, masterSystemTime=1733595316231 2024-12-07T18:15:16,247 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,247 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,248 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6f3992ce41b26fed1c9fca93d87e0364, regionState=OPEN, openSeqNum=5, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:15:16,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-12-07T18:15:16,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 in 171 msec 2024-12-07T18:15:16,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-07T18:15:16,252 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, REOPEN/MOVE in 488 msec 2024-12-07T18:15:16,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-07T18:15:16,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-12-07T18:15:16,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-12-07T18:15:16,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-07T18:15:16,257 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:16,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774ef6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:16,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:16,258 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:16,258 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3335bcf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:16,259 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:16,260 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:16,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65260754, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:16,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:16,261 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67bee2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:16,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:16,262 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:16,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a8c872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:16,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:16,264 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5282c45f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:16,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:16,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:16,265 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:16,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24189c31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:16,266 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:16,267 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbf32a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:16,268 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:16,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7de7c6e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:16,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:16,269 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:16,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@299bcb9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:16,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:16,270 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3acfc38d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,271 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:16,271 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:16,272 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff9bbe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:16,273 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac7abda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:16,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:16,274 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ecb72f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:16,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:16,275 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:16,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c4e7546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:16,276 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:16,277 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21961100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:16,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:16,278 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:16,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e6bd4bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:16,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:16,280 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6047d1f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:16,280 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:16,281 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:16,281 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465e9599, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:16,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:16,282 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf29b5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:16,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:16,283 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:16,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd2ed9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:16,284 DEBUG [Thread-1610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:16,284 DEBUG [Thread-1621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:16,284 DEBUG [Thread-1613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:16,285 DEBUG [Thread-1615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:16,285 DEBUG [Thread-1602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:16,285 DEBUG [Thread-1608 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:16,285 DEBUG [Thread-1617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:16,286 DEBUG [Thread-1619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:16,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:16,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:16,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:16,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:16,287 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:16,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:16,288 DEBUG [Thread-1604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:16,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:16,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:16,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:16,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:16,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-07T18:15:16,289 DEBUG [Thread-1606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:16,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:16,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-07T18:15:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:16,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:15:16,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:16,290 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:15:16,291 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:16,291 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:16,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:15:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:15:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:15:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:15:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:15:16,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:15:16,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:15:16,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:15:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:15:16,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:16,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:16,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:16,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:16,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:16,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:16,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b7213132d8bc4f17a20d46563ab701b8_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595316304/Put/seqid=0 2024-12-07T18:15:16,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742178_1354 (size=14594) 2024-12-07T18:15:16,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595376344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595316339, max timestamp=1733595316355Num uniques 859; 2024-12-07T18:15:16,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:16,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595316339, max timestamp=1733595316355Num uniques 859; 2024-12-07T18:15:16,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:16,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595316339, max timestamp=1733595316355Num uniques 859; 2024-12-07T18:15:16,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-07T18:15:16,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595376392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595376396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595376398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595376404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-07T18:15:16,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:16,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595376468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595376508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595376509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595376518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595376518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-07T18:15:16,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:16,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-07T18:15:16,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595376679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595376728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595376729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-07T18:15:16,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:16,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,752 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:16,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595376739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:16,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595376739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:16,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b7213132d8bc4f17a20d46563ab701b8_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b7213132d8bc4f17a20d46563ab701b8_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:16,767 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/79df6dcd289848cfa30a22f1e6c8ef42, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:16,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/79df6dcd289848cfa30a22f1e6c8ef42 is 175, key is test_row_0/A:col10/1733595316304/Put/seqid=0 2024-12-07T18:15:16,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742179_1355 (size=39549) 2024-12-07T18:15:16,789 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/79df6dcd289848cfa30a22f1e6c8ef42 2024-12-07T18:15:16,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/416f684013bf4c659d6fbd78ba1f51b4 is 50, key is test_row_0/B:col10/1733595316304/Put/seqid=0 2024-12-07T18:15:16,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742180_1356 (size=12001) 2024-12-07T18:15:16,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/416f684013bf4c659d6fbd78ba1f51b4 2024-12-07T18:15:16,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6d0829168b77492bbf22f15441b01e4a is 50, key is test_row_0/C:col10/1733595316304/Put/seqid=0 2024-12-07T18:15:16,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742181_1357 (size=12001) 2024-12-07T18:15:16,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-07T18:15:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:16,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:16,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-07T18:15:17,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595376998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595377048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-07T18:15:17,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:17,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:17,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595377058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595377069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595377078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-07T18:15:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:17,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:17,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:17,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6d0829168b77492bbf22f15441b01e4a 2024-12-07T18:15:17,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/79df6dcd289848cfa30a22f1e6c8ef42 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/79df6dcd289848cfa30a22f1e6c8ef42 2024-12-07T18:15:17,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/79df6dcd289848cfa30a22f1e6c8ef42, entries=200, sequenceid=15, filesize=38.6 K 2024-12-07T18:15:17,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/416f684013bf4c659d6fbd78ba1f51b4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/416f684013bf4c659d6fbd78ba1f51b4 2024-12-07T18:15:17,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/416f684013bf4c659d6fbd78ba1f51b4, entries=150, sequenceid=15, filesize=11.7 K 2024-12-07T18:15:17,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6d0829168b77492bbf22f15441b01e4a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6d0829168b77492bbf22f15441b01e4a 2024-12-07T18:15:17,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6d0829168b77492bbf22f15441b01e4a, entries=150, sequenceid=15, filesize=11.7 K 2024-12-07T18:15:17,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6f3992ce41b26fed1c9fca93d87e0364 in 1024ms, sequenceid=15, compaction requested=false 2024-12-07T18:15:17,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:17,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-07T18:15:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:17,369 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-07T18:15:17,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:17,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:17,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:17,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:17,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:17,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:17,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207bf80eee1760d4e489127a282c3f86d64_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595316394/Put/seqid=0 2024-12-07T18:15:17,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742182_1358 (size=12154) 2024-12-07T18:15:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-07T18:15:17,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:17,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:17,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595377568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595377578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595377578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595377588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595377608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:17,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595377698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:17,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:17,788 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207bf80eee1760d4e489127a282c3f86d64_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207bf80eee1760d4e489127a282c3f86d64_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:17,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/1726a760c8494ac4a6d0d54da370a68d, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:17,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/1726a760c8494ac4a6d0d54da370a68d is 175, key is test_row_0/A:col10/1733595316394/Put/seqid=0 2024-12-07T18:15:17,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742183_1359 (size=30955) 2024-12-07T18:15:17,803 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/1726a760c8494ac4a6d0d54da370a68d 2024-12-07T18:15:17,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/8fdd116bc4d142b592da875c7b63b228 is 50, key is test_row_0/B:col10/1733595316394/Put/seqid=0 2024-12-07T18:15:17,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742184_1360 (size=12001) 2024-12-07T18:15:17,853 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/8fdd116bc4d142b592da875c7b63b228 2024-12-07T18:15:17,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/885e9da2a3bd499387a97f7cb8a0f37d is 50, key is test_row_0/C:col10/1733595316394/Put/seqid=0 2024-12-07T18:15:17,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742185_1361 (size=12001) 2024-12-07T18:15:17,878 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/885e9da2a3bd499387a97f7cb8a0f37d 2024-12-07T18:15:17,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/1726a760c8494ac4a6d0d54da370a68d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1726a760c8494ac4a6d0d54da370a68d 2024-12-07T18:15:17,891 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1726a760c8494ac4a6d0d54da370a68d, entries=150, sequenceid=50, filesize=30.2 K 2024-12-07T18:15:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/8fdd116bc4d142b592da875c7b63b228 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8fdd116bc4d142b592da875c7b63b228 2024-12-07T18:15:17,897 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8fdd116bc4d142b592da875c7b63b228, entries=150, sequenceid=50, filesize=11.7 K 2024-12-07T18:15:17,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/885e9da2a3bd499387a97f7cb8a0f37d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/885e9da2a3bd499387a97f7cb8a0f37d 2024-12-07T18:15:17,903 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/885e9da2a3bd499387a97f7cb8a0f37d, entries=150, sequenceid=50, filesize=11.7 K 2024-12-07T18:15:17,904 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 6f3992ce41b26fed1c9fca93d87e0364 in 534ms, sequenceid=50, compaction requested=false 2024-12-07T18:15:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:17,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-07T18:15:17,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-07T18:15:17,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-07T18:15:17,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-12-07T18:15:17,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.6220 sec 2024-12-07T18:15:17,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:17,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T18:15:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:17,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:17,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:17,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:17,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:17,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:17,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412078f3dbb259d454226b511bb375569099d_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595317923/Put/seqid=0 2024-12-07T18:15:17,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742186_1362 (size=14594) 2024-12-07T18:15:18,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:18,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595378081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:18,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:18,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595378208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:18,343 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:18,347 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412078f3dbb259d454226b511bb375569099d_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412078f3dbb259d454226b511bb375569099d_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:18,348 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/307b30d7222846e8b612222083058705, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:18,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/307b30d7222846e8b612222083058705 is 175, key is test_row_0/A:col10/1733595317923/Put/seqid=0 2024-12-07T18:15:18,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742187_1363 (size=39549) 2024-12-07T18:15:18,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595378419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-07T18:15:18,438 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:18,439 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:18,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-07T18:15:18,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-07T18:15:18,441 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:18,441 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:18,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-07T18:15:18,572 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T18:15:18,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-07T18:15:18,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:18,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:18,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:18,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:18,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:18,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:18,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595378588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:18,601 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:18,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595378609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:18,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595378609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:18,617 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,617 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:18,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595378628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:18,639 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:18,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595378728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:18,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:18,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-07T18:15:18,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:18,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:18,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:18,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:18,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:18,753 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/307b30d7222846e8b612222083058705 2024-12-07T18:15:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-07T18:15:18,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/6443af7d57204b4aa07a7f88eb042088 is 50, key is test_row_0/B:col10/1733595317923/Put/seqid=0 2024-12-07T18:15:18,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742188_1364 (size=12001) 2024-12-07T18:15:18,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-07T18:15:18,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:18,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:18,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:18,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:18,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:19,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-07T18:15:19,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:19,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:19,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:19,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:19,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-07T18:15:19,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/6443af7d57204b4aa07a7f88eb042088 2024-12-07T18:15:19,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/47d65eeab47c40b8bdec62da8baffa43 is 50, key is test_row_0/C:col10/1733595317923/Put/seqid=0 2024-12-07T18:15:19,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742189_1365 (size=12001) 2024-12-07T18:15:19,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/47d65eeab47c40b8bdec62da8baffa43 2024-12-07T18:15:19,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/307b30d7222846e8b612222083058705 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/307b30d7222846e8b612222083058705 2024-12-07T18:15:19,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/307b30d7222846e8b612222083058705, entries=200, sequenceid=62, filesize=38.6 K 2024-12-07T18:15:19,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/6443af7d57204b4aa07a7f88eb042088 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/6443af7d57204b4aa07a7f88eb042088 2024-12-07T18:15:19,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/6443af7d57204b4aa07a7f88eb042088, entries=150, sequenceid=62, filesize=11.7 K 2024-12-07T18:15:19,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/47d65eeab47c40b8bdec62da8baffa43 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/47d65eeab47c40b8bdec62da8baffa43 2024-12-07T18:15:19,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/47d65eeab47c40b8bdec62da8baffa43, entries=150, sequenceid=62, filesize=11.7 K 2024-12-07T18:15:19,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6f3992ce41b26fed1c9fca93d87e0364 in 1271ms, sequenceid=62, compaction requested=true 2024-12-07T18:15:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:19,195 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:19,195 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:19,196 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:19,196 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:19,196 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/A is initiating minor compaction (all files) 2024-12-07T18:15:19,196 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/B is initiating minor compaction (all files) 2024-12-07T18:15:19,196 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/A in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:19,196 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/B in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:19,196 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/79df6dcd289848cfa30a22f1e6c8ef42, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1726a760c8494ac4a6d0d54da370a68d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/307b30d7222846e8b612222083058705] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=107.5 K 2024-12-07T18:15:19,196 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/416f684013bf4c659d6fbd78ba1f51b4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8fdd116bc4d142b592da875c7b63b228, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/6443af7d57204b4aa07a7f88eb042088] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=35.2 K 2024-12-07T18:15:19,196 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:19,197 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/79df6dcd289848cfa30a22f1e6c8ef42, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1726a760c8494ac4a6d0d54da370a68d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/307b30d7222846e8b612222083058705] 2024-12-07T18:15:19,197 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 416f684013bf4c659d6fbd78ba1f51b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733595316293 2024-12-07T18:15:19,197 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79df6dcd289848cfa30a22f1e6c8ef42, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733595316293 2024-12-07T18:15:19,197 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fdd116bc4d142b592da875c7b63b228, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733595316390 2024-12-07T18:15:19,197 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1726a760c8494ac4a6d0d54da370a68d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733595316390 2024-12-07T18:15:19,197 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6443af7d57204b4aa07a7f88eb042088, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733595317528 2024-12-07T18:15:19,198 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 307b30d7222846e8b612222083058705, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733595317528 2024-12-07T18:15:19,205 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:19,207 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#B#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:19,207 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/ed39a0177db14705a6705dc309301c2a is 50, key is test_row_0/B:col10/1733595317923/Put/seqid=0 2024-12-07T18:15:19,207 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207844de67b24d2460696ee3a26522ed3bf_6f3992ce41b26fed1c9fca93d87e0364 store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:19,210 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207844de67b24d2460696ee3a26522ed3bf_6f3992ce41b26fed1c9fca93d87e0364, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:19,210 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207844de67b24d2460696ee3a26522ed3bf_6f3992ce41b26fed1c9fca93d87e0364 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:19,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-07T18:15:19,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:19,211 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-07T18:15:19,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:19,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:19,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:19,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:19,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:19,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:19,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207946c7da4bbf343678acc37a9fce7e6d0_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595318058/Put/seqid=0 2024-12-07T18:15:19,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742190_1366 (size=4469) 2024-12-07T18:15:19,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742191_1367 (size=12104) 2024-12-07T18:15:19,234 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#A#compaction#313 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:19,235 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/9c83e9c373f448c490c8675945bcff82 is 175, key is test_row_0/A:col10/1733595317923/Put/seqid=0 2024-12-07T18:15:19,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742192_1368 (size=12154) 2024-12-07T18:15:19,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742193_1369 (size=31058) 2024-12-07T18:15:19,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:19,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:19,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:19,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595379322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:19,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:19,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595379439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:19,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:19,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:19,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-07T18:15:19,638 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/ed39a0177db14705a6705dc309301c2a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ed39a0177db14705a6705dc309301c2a 2024-12-07T18:15:19,643 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/B of 6f3992ce41b26fed1c9fca93d87e0364 into ed39a0177db14705a6705dc309301c2a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:19,643 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:19,643 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/B, priority=13, startTime=1733595319195; duration=0sec 2024-12-07T18:15:19,643 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:19,643 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:B 2024-12-07T18:15:19,643 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:19,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:19,644 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:19,644 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/C is initiating minor compaction (all files) 2024-12-07T18:15:19,644 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/C in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:19,644 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6d0829168b77492bbf22f15441b01e4a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/885e9da2a3bd499387a97f7cb8a0f37d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/47d65eeab47c40b8bdec62da8baffa43] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=35.2 K 2024-12-07T18:15:19,645 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0829168b77492bbf22f15441b01e4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733595316293 2024-12-07T18:15:19,645 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 885e9da2a3bd499387a97f7cb8a0f37d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733595316390 2024-12-07T18:15:19,646 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d65eeab47c40b8bdec62da8baffa43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733595317528 2024-12-07T18:15:19,647 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207946c7da4bbf343678acc37a9fce7e6d0_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207946c7da4bbf343678acc37a9fce7e6d0_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:19,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/9dbb962ad2fc42b68766266b9bceece8, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:19,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/9dbb962ad2fc42b68766266b9bceece8 is 175, key is test_row_0/A:col10/1733595318058/Put/seqid=0 2024-12-07T18:15:19,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595379649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:19,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:19,662 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/9c83e9c373f448c490c8675945bcff82 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9c83e9c373f448c490c8675945bcff82 2024-12-07T18:15:19,668 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/A of 6f3992ce41b26fed1c9fca93d87e0364 into 9c83e9c373f448c490c8675945bcff82(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:19,668 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:19,668 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/A, priority=13, startTime=1733595319195; duration=0sec 2024-12-07T18:15:19,668 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:19,668 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:A 2024-12-07T18:15:19,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742194_1370 (size=30955) 2024-12-07T18:15:19,678 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:19,678 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/1b9351f1bce343f3a7b5f2734149236d is 50, key is test_row_0/C:col10/1733595317923/Put/seqid=0 2024-12-07T18:15:19,678 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/9dbb962ad2fc42b68766266b9bceece8 2024-12-07T18:15:19,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742195_1371 (size=12104) 2024-12-07T18:15:19,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/0a09572afd9d473eb81a4d6dfd7ea960 is 50, key is test_row_0/B:col10/1733595318058/Put/seqid=0 2024-12-07T18:15:19,691 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/1b9351f1bce343f3a7b5f2734149236d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/1b9351f1bce343f3a7b5f2734149236d 2024-12-07T18:15:19,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742196_1372 (size=12001) 2024-12-07T18:15:19,702 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/C of 6f3992ce41b26fed1c9fca93d87e0364 into 1b9351f1bce343f3a7b5f2734149236d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:19,702 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:19,702 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/C, priority=13, startTime=1733595319195; duration=0sec 2024-12-07T18:15:19,702 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:19,702 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:C 2024-12-07T18:15:19,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:19,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595379968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:19,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:20,101 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/0a09572afd9d473eb81a4d6dfd7ea960 2024-12-07T18:15:20,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/fbf44b8c9472471fa810f4df0a3fb943 is 50, key is test_row_0/C:col10/1733595318058/Put/seqid=0 2024-12-07T18:15:20,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742197_1373 (size=12001) 2024-12-07T18:15:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595380488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:20,517 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/fbf44b8c9472471fa810f4df0a3fb943 2024-12-07T18:15:20,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/9dbb962ad2fc42b68766266b9bceece8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9dbb962ad2fc42b68766266b9bceece8 2024-12-07T18:15:20,525 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9dbb962ad2fc42b68766266b9bceece8, entries=150, sequenceid=82, filesize=30.2 K 2024-12-07T18:15:20,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/0a09572afd9d473eb81a4d6dfd7ea960 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/0a09572afd9d473eb81a4d6dfd7ea960 2024-12-07T18:15:20,531 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/0a09572afd9d473eb81a4d6dfd7ea960, entries=150, sequenceid=82, filesize=11.7 K 2024-12-07T18:15:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/fbf44b8c9472471fa810f4df0a3fb943 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/fbf44b8c9472471fa810f4df0a3fb943 2024-12-07T18:15:20,535 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/fbf44b8c9472471fa810f4df0a3fb943, entries=150, sequenceid=82, filesize=11.7 K 2024-12-07T18:15:20,536 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6f3992ce41b26fed1c9fca93d87e0364 in 1325ms, sequenceid=82, compaction requested=false 2024-12-07T18:15:20,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:20,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:20,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-07T18:15:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-07T18:15:20,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-07T18:15:20,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0960 sec 2024-12-07T18:15:20,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.1010 sec 2024-12-07T18:15:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-07T18:15:20,589 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:20,589 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-07T18:15:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T18:15:20,591 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:20,592 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:20,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:20,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T18:15:20,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:20,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:20,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:20,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:20,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:20,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:20,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120707ee97bd59ba4a0296fe73e3c8a67ec7_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595319312/Put/seqid=0 2024-12-07T18:15:20,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742198_1374 (size=14594) 2024-12-07T18:15:20,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595380669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:20,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595380670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:20,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595380675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:20,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595380685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:20,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T18:15:20,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T18:15:20,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:20,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:20,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:20,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:20,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:20,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595380788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:20,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:20,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595380788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:20,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:20,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595380788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:20,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595380809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:20,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:20,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T18:15:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:20,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:20,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T18:15:21,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595381008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595381008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595381008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595381028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,043 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:21,047 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120707ee97bd59ba4a0296fe73e3c8a67ec7_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120707ee97bd59ba4a0296fe73e3c8a67ec7_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:21,047 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/17ebcf15ce8549048f806f72acca9607, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:21,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/17ebcf15ce8549048f806f72acca9607 is 175, key is test_row_0/A:col10/1733595319312/Put/seqid=0 2024-12-07T18:15:21,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742199_1375 (size=39549) 2024-12-07T18:15:21,052 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/17ebcf15ce8549048f806f72acca9607 2024-12-07T18:15:21,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T18:15:21,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:21,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/24b33633582f4d04b59a65d85aad9c9d is 50, key is test_row_0/B:col10/1733595319312/Put/seqid=0 2024-12-07T18:15:21,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742200_1376 (size=12001) 2024-12-07T18:15:21,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T18:15:21,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:21,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T18:15:21,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595381328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595381328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595381328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T18:15:21,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:21,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595381349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/24b33633582f4d04b59a65d85aad9c9d 2024-12-07T18:15:21,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/3b2b15478a804249bed86b869c9ab0d9 is 50, key is test_row_0/C:col10/1733595319312/Put/seqid=0 2024-12-07T18:15:21,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742201_1377 (size=12001) 2024-12-07T18:15:21,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595381498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,506 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:21,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T18:15:21,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:21,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T18:15:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:21,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T18:15:21,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T18:15:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:21,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:21,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595381848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595381849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595381849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595381868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:21,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/3b2b15478a804249bed86b869c9ab0d9 2024-12-07T18:15:21,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/17ebcf15ce8549048f806f72acca9607 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/17ebcf15ce8549048f806f72acca9607 2024-12-07T18:15:21,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/17ebcf15ce8549048f806f72acca9607, entries=200, sequenceid=98, filesize=38.6 K 2024-12-07T18:15:21,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/24b33633582f4d04b59a65d85aad9c9d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/24b33633582f4d04b59a65d85aad9c9d 2024-12-07T18:15:21,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/24b33633582f4d04b59a65d85aad9c9d, entries=150, sequenceid=98, filesize=11.7 K 2024-12-07T18:15:21,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/3b2b15478a804249bed86b869c9ab0d9 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3b2b15478a804249bed86b869c9ab0d9 2024-12-07T18:15:21,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3b2b15478a804249bed86b869c9ab0d9, entries=150, sequenceid=98, filesize=11.7 K 2024-12-07T18:15:21,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6f3992ce41b26fed1c9fca93d87e0364 in 1272ms, sequenceid=98, compaction requested=true 2024-12-07T18:15:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:21,895 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:21,895 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:21,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:21,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:21,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:21,896 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:21,896 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/B is initiating minor compaction (all files) 2024-12-07T18:15:21,896 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/B in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,896 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:21,896 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ed39a0177db14705a6705dc309301c2a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/0a09572afd9d473eb81a4d6dfd7ea960, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/24b33633582f4d04b59a65d85aad9c9d] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=35.3 K 2024-12-07T18:15:21,897 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/A is initiating minor compaction (all files) 2024-12-07T18:15:21,897 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/A in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,897 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9c83e9c373f448c490c8675945bcff82, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9dbb962ad2fc42b68766266b9bceece8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/17ebcf15ce8549048f806f72acca9607] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=99.2 K 2024-12-07T18:15:21,897 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,897 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9c83e9c373f448c490c8675945bcff82, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9dbb962ad2fc42b68766266b9bceece8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/17ebcf15ce8549048f806f72acca9607] 2024-12-07T18:15:21,897 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ed39a0177db14705a6705dc309301c2a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733595317528 2024-12-07T18:15:21,897 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c83e9c373f448c490c8675945bcff82, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733595317528 2024-12-07T18:15:21,898 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a09572afd9d473eb81a4d6dfd7ea960, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733595318015 2024-12-07T18:15:21,898 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 24b33633582f4d04b59a65d85aad9c9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733595319312 2024-12-07T18:15:21,898 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dbb962ad2fc42b68766266b9bceece8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733595318015 2024-12-07T18:15:21,899 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17ebcf15ce8549048f806f72acca9607, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733595319302 2024-12-07T18:15:21,905 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:21,905 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:21,906 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/f404a08b5fa943cb8f79f874a7dfb2c0 is 50, key is test_row_0/B:col10/1733595319312/Put/seqid=0 2024-12-07T18:15:21,908 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207e87e588ed051413282eca85f005c3f94_6f3992ce41b26fed1c9fca93d87e0364 store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:21,910 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207e87e588ed051413282eca85f005c3f94_6f3992ce41b26fed1c9fca93d87e0364, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:21,910 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e87e588ed051413282eca85f005c3f94_6f3992ce41b26fed1c9fca93d87e0364 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:21,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742202_1378 (size=12207) 2024-12-07T18:15:21,921 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/f404a08b5fa943cb8f79f874a7dfb2c0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/f404a08b5fa943cb8f79f874a7dfb2c0 2024-12-07T18:15:21,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742203_1379 (size=4469) 2024-12-07T18:15:21,928 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/B of 6f3992ce41b26fed1c9fca93d87e0364 into f404a08b5fa943cb8f79f874a7dfb2c0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:21,928 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:21,928 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/B, priority=13, startTime=1733595321895; duration=0sec 2024-12-07T18:15:21,928 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:21,928 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:B 2024-12-07T18:15:21,928 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:21,929 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:21,929 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/C is initiating minor compaction (all files) 2024-12-07T18:15:21,929 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/C in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,929 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/1b9351f1bce343f3a7b5f2734149236d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/fbf44b8c9472471fa810f4df0a3fb943, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3b2b15478a804249bed86b869c9ab0d9] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=35.3 K 2024-12-07T18:15:21,929 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b9351f1bce343f3a7b5f2734149236d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733595317528 2024-12-07T18:15:21,930 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting fbf44b8c9472471fa810f4df0a3fb943, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733595318015 2024-12-07T18:15:21,930 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b2b15478a804249bed86b869c9ab0d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733595319312 2024-12-07T18:15:21,937 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:21,937 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/81ce03d58f3b4c3abf225762084367ff is 50, key is test_row_0/C:col10/1733595319312/Put/seqid=0 2024-12-07T18:15:21,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742204_1380 (size=12207) 2024-12-07T18:15:21,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-07T18:15:21,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:21,970 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T18:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:21,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:21,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412078260765c80574cecb1855e0796ce8151_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595320669/Put/seqid=0 2024-12-07T18:15:21,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742205_1381 (size=12154) 2024-12-07T18:15:21,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:21,989 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412078260765c80574cecb1855e0796ce8151_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412078260765c80574cecb1855e0796ce8151_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/7a180b12a7b24cc4a2e3f788520b853c, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:21,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/7a180b12a7b24cc4a2e3f788520b853c is 175, key is test_row_0/A:col10/1733595320669/Put/seqid=0 2024-12-07T18:15:21,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742206_1382 (size=30955) 2024-12-07T18:15:22,326 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#A#compaction#322 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:22,326 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/a2d3eea46ec14a409c172267bd8e674e is 175, key is test_row_0/A:col10/1733595319312/Put/seqid=0 2024-12-07T18:15:22,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742207_1383 (size=31161) 2024-12-07T18:15:22,347 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/81ce03d58f3b4c3abf225762084367ff as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/81ce03d58f3b4c3abf225762084367ff 2024-12-07T18:15:22,351 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/C of 6f3992ce41b26fed1c9fca93d87e0364 into 81ce03d58f3b4c3abf225762084367ff(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:22,351 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:22,351 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/C, priority=13, startTime=1733595321895; duration=0sec 2024-12-07T18:15:22,351 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:22,351 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:C 2024-12-07T18:15:22,400 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/7a180b12a7b24cc4a2e3f788520b853c 2024-12-07T18:15:22,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/ea8b82784efa4737b97e19cf531021ab is 50, key is test_row_0/B:col10/1733595320669/Put/seqid=0 2024-12-07T18:15:22,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742208_1384 (size=12001) 2024-12-07T18:15:22,426 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/ea8b82784efa4737b97e19cf531021ab 2024-12-07T18:15:22,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/219b1c1a850843e285075d514dcdd5d5 is 50, key is test_row_0/C:col10/1733595320669/Put/seqid=0 2024-12-07T18:15:22,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742209_1385 (size=12001) 2024-12-07T18:15:22,447 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/219b1c1a850843e285075d514dcdd5d5 2024-12-07T18:15:22,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/7a180b12a7b24cc4a2e3f788520b853c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7a180b12a7b24cc4a2e3f788520b853c 2024-12-07T18:15:22,455 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7a180b12a7b24cc4a2e3f788520b853c, entries=150, sequenceid=118, filesize=30.2 K 2024-12-07T18:15:22,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/ea8b82784efa4737b97e19cf531021ab as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ea8b82784efa4737b97e19cf531021ab 2024-12-07T18:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,459 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ea8b82784efa4737b97e19cf531021ab, entries=150, sequenceid=118, filesize=11.7 K 2024-12-07T18:15:22,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/219b1c1a850843e285075d514dcdd5d5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/219b1c1a850843e285075d514dcdd5d5 2024-12-07T18:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,463 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/219b1c1a850843e285075d514dcdd5d5, entries=150, sequenceid=118, filesize=11.7 K 2024-12-07T18:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,464 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 6f3992ce41b26fed1c9fca93d87e0364 in 495ms, sequenceid=118, compaction requested=false 2024-12-07T18:15:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-07T18:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-07T18:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-07T18:15:22,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8730 sec 2024-12-07T18:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,470 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.8800 sec 2024-12-07T18:15:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,736 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/a2d3eea46ec14a409c172267bd8e674e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a2d3eea46ec14a409c172267bd8e674e 2024-12-07T18:15:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-07T18:15:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,738 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,739 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:22,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-07T18:15:22,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,742 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-07T18:15:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,742 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/A of 6f3992ce41b26fed1c9fca93d87e0364 into a2d3eea46ec14a409c172267bd8e674e(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:22,742 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:22,742 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/A, priority=13, startTime=1733595321895; duration=0sec 2024-12-07T18:15:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,742 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:22,742 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:A 2024-12-07T18:15:22,743 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-07T18:15:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-07T18:15:22,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:22,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-07T18:15:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-07T18:15:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-07T18:15:22,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-12-07T18:15:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,902 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 162 msec 2024-12-07T18:15:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:22,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:15:22,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:22,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120739f39b036123419db5781bf034a1484b_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595322986/Put/seqid=0 2024-12-07T18:15:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742210_1386 (size=22068) 2024-12-07T18:15:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,016 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120739f39b036123419db5781bf034a1484b_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120739f39b036123419db5781bf034a1484b_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/a6d81a195735467da1a93e531b4ca472, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/a6d81a195735467da1a93e531b4ca472 is 175, key is test_row_0/A:col10/1733595322986/Put/seqid=0 2024-12-07T18:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742211_1387 (size=65473) 2024-12-07T18:15:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-07T18:15:23,059 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:23,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,059 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-07T18:15:23,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T18:15:23,061 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,062 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:23,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:23,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:23,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595323070, max timestamp=1733595323094Num uniques 859; 2024-12-07T18:15:23,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:15:23,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595323070, max timestamp=1733595323094Num uniques 859; 2024-12-07T18:15:23,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595323070, max timestamp=1733595323094Num uniques 859; 2024-12-07T18:15:23,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:15:23,099 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:15:23,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T18:15:23,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595383163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595383164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595383170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595383183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,201 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T18:15:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595383288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595383288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595383288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595383308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T18:15:23,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T18:15:23,434 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/a6d81a195735467da1a93e531b4ca472 2024-12-07T18:15:23,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/71a7573988944da284371daebea23b8c is 50, key is test_row_0/B:col10/1733595322986/Put/seqid=0 2024-12-07T18:15:23,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742212_1388 (size=12101) 2024-12-07T18:15:23,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595383508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595383508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T18:15:23,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595383508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:23,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595383529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595383529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,539 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T18:15:23,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:23,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T18:15:23,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T18:15:23,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:23,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595383828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595383828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595383828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/71a7573988944da284371daebea23b8c 2024-12-07T18:15:23,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6e60a72c17e24580ab83b553dff807fb is 50, key is test_row_0/C:col10/1733595322986/Put/seqid=0 2024-12-07T18:15:23,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742213_1389 (size=12101) 2024-12-07T18:15:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595383849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:23,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T18:15:23,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:23,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:23,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:23,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:24,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T18:15:24,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:24,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:24,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:24,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:24,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:24,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T18:15:24,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6e60a72c17e24580ab83b553dff807fb 2024-12-07T18:15:24,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/a6d81a195735467da1a93e531b4ca472 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a6d81a195735467da1a93e531b4ca472 2024-12-07T18:15:24,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a6d81a195735467da1a93e531b4ca472, entries=350, sequenceid=131, filesize=63.9 K 2024-12-07T18:15:24,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/71a7573988944da284371daebea23b8c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/71a7573988944da284371daebea23b8c 2024-12-07T18:15:24,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/71a7573988944da284371daebea23b8c, entries=150, sequenceid=131, filesize=11.8 K 2024-12-07T18:15:24,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6e60a72c17e24580ab83b553dff807fb as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e60a72c17e24580ab83b553dff807fb 2024-12-07T18:15:24,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e60a72c17e24580ab83b553dff807fb, entries=150, sequenceid=131, filesize=11.8 K 2024-12-07T18:15:24,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6f3992ce41b26fed1c9fca93d87e0364 in 1286ms, sequenceid=131, compaction requested=true 2024-12-07T18:15:24,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:24,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:24,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:24,276 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:24,276 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:24,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:24,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:24,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:24,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:24,277 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:24,277 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/A is initiating minor compaction (all files) 2024-12-07T18:15:24,278 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/A in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:24,278 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a2d3eea46ec14a409c172267bd8e674e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7a180b12a7b24cc4a2e3f788520b853c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a6d81a195735467da1a93e531b4ca472] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=124.6 K 2024-12-07T18:15:24,278 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:24,278 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a2d3eea46ec14a409c172267bd8e674e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7a180b12a7b24cc4a2e3f788520b853c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a6d81a195735467da1a93e531b4ca472] 2024-12-07T18:15:24,278 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:24,278 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/B is initiating minor compaction (all files) 2024-12-07T18:15:24,278 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/B in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:24,278 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/f404a08b5fa943cb8f79f874a7dfb2c0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ea8b82784efa4737b97e19cf531021ab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/71a7573988944da284371daebea23b8c] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=35.5 K 2024-12-07T18:15:24,279 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2d3eea46ec14a409c172267bd8e674e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733595319312 2024-12-07T18:15:24,279 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting f404a08b5fa943cb8f79f874a7dfb2c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733595319312 2024-12-07T18:15:24,279 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a180b12a7b24cc4a2e3f788520b853c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733595320651 2024-12-07T18:15:24,279 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ea8b82784efa4737b97e19cf531021ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733595320651 2024-12-07T18:15:24,280 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6d81a195735467da1a93e531b4ca472, keycount=350, bloomtype=ROW, size=63.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595322965 2024-12-07T18:15:24,281 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a7573988944da284371daebea23b8c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595322966 2024-12-07T18:15:24,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-07T18:15:24,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:24,288 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-07T18:15:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:24,290 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:24,294 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#B#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:24,296 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/51aa6fd313804281bff7a10c347a9e08 is 50, key is test_row_0/B:col10/1733595322986/Put/seqid=0 2024-12-07T18:15:24,301 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120716330fb3ff41410c9e48c40f77bc4af0_6f3992ce41b26fed1c9fca93d87e0364 store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:24,303 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120716330fb3ff41410c9e48c40f77bc4af0_6f3992ce41b26fed1c9fca93d87e0364, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:24,303 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120716330fb3ff41410c9e48c40f77bc4af0_6f3992ce41b26fed1c9fca93d87e0364 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:24,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742214_1390 (size=12409) 2024-12-07T18:15:24,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412079fc10ae70c9d4fcda6266cbf559c0b33_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595323181/Put/seqid=0 2024-12-07T18:15:24,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742215_1391 (size=4469) 2024-12-07T18:15:24,323 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#A#compaction#330 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:24,323 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/e60bde73cc8642b1a3bd64ba7d7a52ad is 175, key is test_row_0/A:col10/1733595322986/Put/seqid=0 2024-12-07T18:15:24,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742216_1392 (size=12304) 2024-12-07T18:15:24,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:24,332 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412079fc10ae70c9d4fcda6266cbf559c0b33_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412079fc10ae70c9d4fcda6266cbf559c0b33_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:24,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/ccdde20de656437daf7555cc88639211, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:24,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/ccdde20de656437daf7555cc88639211 is 175, key is test_row_0/A:col10/1733595323181/Put/seqid=0 2024-12-07T18:15:24,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742217_1393 (size=31363) 2024-12-07T18:15:24,347 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/e60bde73cc8642b1a3bd64ba7d7a52ad as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e60bde73cc8642b1a3bd64ba7d7a52ad 2024-12-07T18:15:24,358 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/A of 6f3992ce41b26fed1c9fca93d87e0364 into e60bde73cc8642b1a3bd64ba7d7a52ad(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:24,358 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:24,358 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/A, priority=13, startTime=1733595324276; duration=0sec 2024-12-07T18:15:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:24,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:24,359 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:24,359 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:A 2024-12-07T18:15:24,359 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:24,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742218_1394 (size=31105) 2024-12-07T18:15:24,361 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/ccdde20de656437daf7555cc88639211 2024-12-07T18:15:24,362 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:24,362 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/C is initiating minor compaction (all files) 2024-12-07T18:15:24,362 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/C in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:24,362 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/81ce03d58f3b4c3abf225762084367ff, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/219b1c1a850843e285075d514dcdd5d5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e60a72c17e24580ab83b553dff807fb] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=35.5 K 2024-12-07T18:15:24,366 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ce03d58f3b4c3abf225762084367ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733595319312 2024-12-07T18:15:24,369 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 219b1c1a850843e285075d514dcdd5d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733595320651 2024-12-07T18:15:24,370 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e60a72c17e24580ab83b553dff807fb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595322966 2024-12-07T18:15:24,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/b858ec743fe146a1b07810d95369ccc1 is 50, key is test_row_0/B:col10/1733595323181/Put/seqid=0 2024-12-07T18:15:24,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595384371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595384373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595384380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,398 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#C#compaction#334 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:24,400 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/59fb1951bd924212b1a949722845e542 is 50, key is test_row_0/C:col10/1733595322986/Put/seqid=0 2024-12-07T18:15:24,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595384385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742219_1395 (size=12151) 2024-12-07T18:15:24,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742220_1396 (size=12409) 2024-12-07T18:15:24,426 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/59fb1951bd924212b1a949722845e542 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/59fb1951bd924212b1a949722845e542 2024-12-07T18:15:24,431 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/C of 6f3992ce41b26fed1c9fca93d87e0364 into 59fb1951bd924212b1a949722845e542(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:24,431 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:24,431 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/C, priority=13, startTime=1733595324277; duration=0sec 2024-12-07T18:15:24,431 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:24,431 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:C 2024-12-07T18:15:24,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595384498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595384498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595384498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595384509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,519 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,710 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/51aa6fd313804281bff7a10c347a9e08 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/51aa6fd313804281bff7a10c347a9e08 2024-12-07T18:15:24,714 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/B of 6f3992ce41b26fed1c9fca93d87e0364 into 51aa6fd313804281bff7a10c347a9e08(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:24,715 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:24,715 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/B, priority=13, startTime=1733595324276; duration=0sec 2024-12-07T18:15:24,715 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:24,715 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:B 2024-12-07T18:15:24,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595384719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595384719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595384719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:24,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595384728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:24,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:24,804 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/b858ec743fe146a1b07810d95369ccc1 2024-12-07T18:15:24,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/579bb8ef40a74793b0b51fbd87f31743 is 50, key is test_row_0/C:col10/1733595323181/Put/seqid=0 2024-12-07T18:15:24,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742221_1397 (size=12151) 2024-12-07T18:15:25,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595385038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595385038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595385038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595385049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T18:15:25,219 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/579bb8ef40a74793b0b51fbd87f31743 2024-12-07T18:15:25,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/ccdde20de656437daf7555cc88639211 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/ccdde20de656437daf7555cc88639211 2024-12-07T18:15:25,226 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/ccdde20de656437daf7555cc88639211, entries=150, sequenceid=166, filesize=30.4 K 2024-12-07T18:15:25,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/b858ec743fe146a1b07810d95369ccc1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/b858ec743fe146a1b07810d95369ccc1 2024-12-07T18:15:25,231 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/b858ec743fe146a1b07810d95369ccc1, entries=150, sequenceid=166, filesize=11.9 K 2024-12-07T18:15:25,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/579bb8ef40a74793b0b51fbd87f31743 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/579bb8ef40a74793b0b51fbd87f31743 2024-12-07T18:15:25,235 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/579bb8ef40a74793b0b51fbd87f31743, entries=150, sequenceid=166, filesize=11.9 K 2024-12-07T18:15:25,236 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 6f3992ce41b26fed1c9fca93d87e0364 in 948ms, sequenceid=166, compaction requested=false 2024-12-07T18:15:25,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:25,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:25,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-07T18:15:25,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-07T18:15:25,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-07T18:15:25,240 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1760 sec 2024-12-07T18:15:25,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1800 sec 2024-12-07T18:15:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:25,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T18:15:25,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:25,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:25,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207f35434651034405dab91ef15b955d26a_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595324381/Put/seqid=0 2024-12-07T18:15:25,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742222_1398 (size=14794) 2024-12-07T18:15:25,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595325593, max timestamp=1733595325612Num uniques 859; 2024-12-07T18:15:25,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595325593, max timestamp=1733595325612Num uniques 859; 2024-12-07T18:15:25,615 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:15:25,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:15:25,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595325593, max timestamp=1733595325612Num uniques 859; 2024-12-07T18:15:25,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:15:25,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595385681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595385681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595385683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595385683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595385808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595385808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595385808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595385809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:25,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:25,970 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:25,974 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207f35434651034405dab91ef15b955d26a_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207f35434651034405dab91ef15b955d26a_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:25,975 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2a2afdd3fe3b43bc988dea13234fb416, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:25,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2a2afdd3fe3b43bc988dea13234fb416 is 175, key is test_row_0/A:col10/1733595324381/Put/seqid=0 2024-12-07T18:15:25,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742223_1399 (size=39749) 2024-12-07T18:15:26,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595386018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595386018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595386018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595386028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595386338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595386338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595386338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595386348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,356 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,381 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2a2afdd3fe3b43bc988dea13234fb416 2024-12-07T18:15:26,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/04466b563e284746b2d23edbd1b88f5b is 50, key is test_row_0/B:col10/1733595324381/Put/seqid=0 2024-12-07T18:15:26,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742224_1400 (size=12151) 2024-12-07T18:15:26,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/04466b563e284746b2d23edbd1b88f5b 2024-12-07T18:15:26,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/3577053d276e41108a435a45d59c7054 is 50, key is test_row_0/C:col10/1733595324381/Put/seqid=0 2024-12-07T18:15:26,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742225_1401 (size=12151) 2024-12-07T18:15:26,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/3577053d276e41108a435a45d59c7054 2024-12-07T18:15:26,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2a2afdd3fe3b43bc988dea13234fb416 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2a2afdd3fe3b43bc988dea13234fb416 2024-12-07T18:15:26,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2a2afdd3fe3b43bc988dea13234fb416, entries=200, sequenceid=180, filesize=38.8 K 2024-12-07T18:15:26,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/04466b563e284746b2d23edbd1b88f5b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/04466b563e284746b2d23edbd1b88f5b 2024-12-07T18:15:26,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/04466b563e284746b2d23edbd1b88f5b, entries=150, sequenceid=180, filesize=11.9 K 2024-12-07T18:15:26,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/3577053d276e41108a435a45d59c7054 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3577053d276e41108a435a45d59c7054 2024-12-07T18:15:26,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3577053d276e41108a435a45d59c7054, entries=150, sequenceid=180, filesize=11.9 K 2024-12-07T18:15:26,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 6f3992ce41b26fed1c9fca93d87e0364 in 876ms, sequenceid=180, compaction requested=true 2024-12-07T18:15:26,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:26,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:26,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:26,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:26,432 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:26,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:26,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:26,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:26,432 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:26,435 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:26,435 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/A is initiating minor compaction (all files) 2024-12-07T18:15:26,435 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/A in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:26,435 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e60bde73cc8642b1a3bd64ba7d7a52ad, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/ccdde20de656437daf7555cc88639211, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2a2afdd3fe3b43bc988dea13234fb416] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=99.8 K 2024-12-07T18:15:26,435 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:26,435 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:26,435 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e60bde73cc8642b1a3bd64ba7d7a52ad, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/ccdde20de656437daf7555cc88639211, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2a2afdd3fe3b43bc988dea13234fb416] 2024-12-07T18:15:26,435 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/B is initiating minor compaction (all files) 2024-12-07T18:15:26,435 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/B in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:26,436 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/51aa6fd313804281bff7a10c347a9e08, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/b858ec743fe146a1b07810d95369ccc1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/04466b563e284746b2d23edbd1b88f5b] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=35.9 K 2024-12-07T18:15:26,436 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting e60bde73cc8642b1a3bd64ba7d7a52ad, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595322966 2024-12-07T18:15:26,436 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 51aa6fd313804281bff7a10c347a9e08, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595322966 2024-12-07T18:15:26,436 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccdde20de656437daf7555cc88639211, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733595323162 2024-12-07T18:15:26,436 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting b858ec743fe146a1b07810d95369ccc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733595323162 2024-12-07T18:15:26,437 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a2afdd3fe3b43bc988dea13234fb416, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733595324358 2024-12-07T18:15:26,437 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 04466b563e284746b2d23edbd1b88f5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733595324377 2024-12-07T18:15:26,443 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:26,444 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120749fdce565f5c4a8c9a6ccd424a08a0ea_6f3992ce41b26fed1c9fca93d87e0364 store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:26,444 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:26,445 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/45ef4496ba6f43cda6cce9aecbae66f0 is 50, key is test_row_0/B:col10/1733595324381/Put/seqid=0 2024-12-07T18:15:26,446 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120749fdce565f5c4a8c9a6ccd424a08a0ea_6f3992ce41b26fed1c9fca93d87e0364, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:26,446 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120749fdce565f5c4a8c9a6ccd424a08a0ea_6f3992ce41b26fed1c9fca93d87e0364 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:26,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742226_1402 (size=12561) 2024-12-07T18:15:26,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742227_1403 (size=4469) 2024-12-07T18:15:26,852 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:26,853 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/9f276732d2154c539be64b4b2ccdfe53 is 175, key is test_row_0/A:col10/1733595324381/Put/seqid=0 2024-12-07T18:15:26,859 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/45ef4496ba6f43cda6cce9aecbae66f0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/45ef4496ba6f43cda6cce9aecbae66f0 2024-12-07T18:15:26,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742228_1404 (size=31515) 2024-12-07T18:15:26,865 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/B of 6f3992ce41b26fed1c9fca93d87e0364 into 45ef4496ba6f43cda6cce9aecbae66f0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:26,865 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:26,865 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/B, priority=13, startTime=1733595326432; duration=0sec 2024-12-07T18:15:26,865 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:26,865 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:B 2024-12-07T18:15:26,865 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:26,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-07T18:15:26,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:26,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:26,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:26,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:26,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:26,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:26,869 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:26,869 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/C is initiating minor compaction (all files) 2024-12-07T18:15:26,869 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/C in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:26,869 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/59fb1951bd924212b1a949722845e542, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/579bb8ef40a74793b0b51fbd87f31743, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3577053d276e41108a435a45d59c7054] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=35.9 K 2024-12-07T18:15:26,870 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/9f276732d2154c539be64b4b2ccdfe53 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9f276732d2154c539be64b4b2ccdfe53 2024-12-07T18:15:26,870 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 59fb1951bd924212b1a949722845e542, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595322966 2024-12-07T18:15:26,871 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 579bb8ef40a74793b0b51fbd87f31743, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733595323162 2024-12-07T18:15:26,871 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3577053d276e41108a435a45d59c7054, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733595324377 2024-12-07T18:15:26,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120747dc412aa09a4a1a9951ea3dbf9f1d3d_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595326864/Put/seqid=0 2024-12-07T18:15:26,879 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/A of 6f3992ce41b26fed1c9fca93d87e0364 into 9f276732d2154c539be64b4b2ccdfe53(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:26,879 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:26,879 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/A, priority=13, startTime=1733595326432; duration=0sec 2024-12-07T18:15:26,879 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:26,879 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:A 2024-12-07T18:15:26,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595386886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742229_1405 (size=14794) 2024-12-07T18:15:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:26,893 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:26,894 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120747dc412aa09a4a1a9951ea3dbf9f1d3d_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120747dc412aa09a4a1a9951ea3dbf9f1d3d_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:26,894 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/b53761101050478884c911d71d76d848 is 50, key is test_row_0/C:col10/1733595324381/Put/seqid=0 2024-12-07T18:15:26,894 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:26,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc is 175, key is test_row_0/A:col10/1733595326864/Put/seqid=0 2024-12-07T18:15:26,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742230_1406 (size=12561) 2024-12-07T18:15:26,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595386888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595386888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595386889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:26,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742231_1407 (size=39749) 2024-12-07T18:15:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:26,906 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc 2024-12-07T18:15:26,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/4b4230c9e50b49b0bf412fac571e1a59 is 50, key is test_row_0/B:col10/1733595326864/Put/seqid=0 2024-12-07T18:15:26,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742232_1408 (size=12151) 2024-12-07T18:15:27,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595387002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595387009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595387009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595387009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-07T18:15:27,219 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:27,219 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-07T18:15:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-07T18:15:27,221 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:27,222 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:27,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:27,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595387219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595387219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595387228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595387228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,303 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/b53761101050478884c911d71d76d848 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/b53761101050478884c911d71d76d848 2024-12-07T18:15:27,308 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/C of 6f3992ce41b26fed1c9fca93d87e0364 into b53761101050478884c911d71d76d848(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:27,308 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:27,308 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/C, priority=13, startTime=1733595326432; duration=0sec 2024-12-07T18:15:27,308 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:27,308 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:C 2024-12-07T18:15:27,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/4b4230c9e50b49b0bf412fac571e1a59 2024-12-07T18:15:27,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/ef31241a21e14b198de2358e9a7e890b is 50, key is test_row_0/C:col10/1733595326864/Put/seqid=0 2024-12-07T18:15:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-07T18:15:27,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742233_1409 (size=12151) 2024-12-07T18:15:27,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/ef31241a21e14b198de2358e9a7e890b 2024-12-07T18:15:27,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc 2024-12-07T18:15:27,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc, entries=200, sequenceid=217, filesize=38.8 K 2024-12-07T18:15:27,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/4b4230c9e50b49b0bf412fac571e1a59 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/4b4230c9e50b49b0bf412fac571e1a59 2024-12-07T18:15:27,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/4b4230c9e50b49b0bf412fac571e1a59, entries=150, sequenceid=217, filesize=11.9 K 2024-12-07T18:15:27,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/ef31241a21e14b198de2358e9a7e890b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/ef31241a21e14b198de2358e9a7e890b 2024-12-07T18:15:27,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/ef31241a21e14b198de2358e9a7e890b, entries=150, sequenceid=217, filesize=11.9 K 2024-12-07T18:15:27,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for 6f3992ce41b26fed1c9fca93d87e0364 in 483ms, sequenceid=217, compaction requested=false 2024-12-07T18:15:27,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:27,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-07T18:15:27,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:27,374 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:15:27,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:27,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:27,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:27,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:27,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:27,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:27,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207103f6a3c533a4e2c9f43e89daa3c29c9_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595326886/Put/seqid=0 2024-12-07T18:15:27,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742234_1410 (size=12304) 2024-12-07T18:15:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-07T18:15:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:27,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:27,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595327565, max timestamp=1733595327580Num uniques 859; 2024-12-07T18:15:27,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:27,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595327565, max timestamp=1733595327580Num uniques 859; 2024-12-07T18:15:27,583 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:27,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595327565, max timestamp=1733595327580Num uniques 859; 2024-12-07T18:15:27,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:27,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595387610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595387611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595387625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595387626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595387626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595387729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595387729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595387748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595387748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595387748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:27,788 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207103f6a3c533a4e2c9f43e89daa3c29c9_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207103f6a3c533a4e2c9f43e89daa3c29c9_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:27,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/12796069dd494c82a5556d3f73f1c0a5, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:27,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/12796069dd494c82a5556d3f73f1c0a5 is 175, key is test_row_0/A:col10/1733595326886/Put/seqid=0 2024-12-07T18:15:27,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742235_1411 (size=31105) 2024-12-07T18:15:27,795 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/12796069dd494c82a5556d3f73f1c0a5 2024-12-07T18:15:27,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/89a2a6d5aa494de6a12496850578654d is 50, key is test_row_0/B:col10/1733595326886/Put/seqid=0 2024-12-07T18:15:27,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742236_1412 (size=12151) 2024-12-07T18:15:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-07T18:15:27,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595387948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595387948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595387968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595387968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595387969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:27,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:27,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:27,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,206 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/89a2a6d5aa494de6a12496850578654d 2024-12-07T18:15:28,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/402316b596fe4365b68836ebe736c550 is 50, key is test_row_0/C:col10/1733595326886/Put/seqid=0 2024-12-07T18:15:28,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742237_1413 (size=12151) 2024-12-07T18:15:28,264 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/402316b596fe4365b68836ebe736c550 2024-12-07T18:15:28,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/12796069dd494c82a5556d3f73f1c0a5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/12796069dd494c82a5556d3f73f1c0a5 2024-12-07T18:15:28,276 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/12796069dd494c82a5556d3f73f1c0a5, entries=150, sequenceid=229, filesize=30.4 K 2024-12-07T18:15:28,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/89a2a6d5aa494de6a12496850578654d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/89a2a6d5aa494de6a12496850578654d 2024-12-07T18:15:28,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595388268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595388269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,285 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/89a2a6d5aa494de6a12496850578654d, entries=150, sequenceid=229, filesize=11.9 K 2024-12-07T18:15:28,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/402316b596fe4365b68836ebe736c550 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/402316b596fe4365b68836ebe736c550 2024-12-07T18:15:28,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595388289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595388289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595388289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,303 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/402316b596fe4365b68836ebe736c550, entries=150, sequenceid=229, filesize=11.9 K 2024-12-07T18:15:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,310 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6f3992ce41b26fed1c9fca93d87e0364 in 936ms, sequenceid=229, compaction requested=true 2024-12-07T18:15:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:28,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-07T18:15:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-07T18:15:28,314 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-07T18:15:28,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0900 sec 2024-12-07T18:15:28,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.0960 sec 2024-12-07T18:15:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-07T18:15:28,358 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:28,359 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-07T18:15:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T18:15:28,361 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:28,361 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:28,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T18:15:28,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-07T18:15:28,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:28,514 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-07T18:15:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:28,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a92b7b39388d47ecb3163bd111f7e333_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595327610/Put/seqid=0 2024-12-07T18:15:28,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742238_1414 (size=12454) 2024-12-07T18:15:28,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:28,535 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a92b7b39388d47ecb3163bd111f7e333_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a92b7b39388d47ecb3163bd111f7e333_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:28,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/98946b8cbfdd4015a859820228ee8e10, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:28,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/98946b8cbfdd4015a859820228ee8e10 is 175, key is test_row_0/A:col10/1733595327610/Put/seqid=0 2024-12-07T18:15:28,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742239_1415 (size=31255) 2024-12-07T18:15:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T18:15:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:28,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:28,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595388808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595388809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595388810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595388820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595388821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595388928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595388928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595388938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,946 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/98946b8cbfdd4015a859820228ee8e10 2024-12-07T18:15:28,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/490091c4fe4845da88dfe1cd3bc3c264 is 50, key is test_row_0/B:col10/1733595327610/Put/seqid=0 2024-12-07T18:15:28,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742240_1416 (size=12301) 2024-12-07T18:15:28,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595388948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T18:15:29,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595389148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:29,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595389148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595389149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:29,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595389168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:29,357 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/490091c4fe4845da88dfe1cd3bc3c264 2024-12-07T18:15:29,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/a2d40686099e4de9bb9226b8f8bb08b7 is 50, key is test_row_0/C:col10/1733595327610/Put/seqid=0 2024-12-07T18:15:29,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742241_1417 (size=12301) 2024-12-07T18:15:29,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595389468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:29,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595389468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595389468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T18:15:29,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595389489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:29,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:29,769 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/a2d40686099e4de9bb9226b8f8bb08b7 2024-12-07T18:15:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/98946b8cbfdd4015a859820228ee8e10 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/98946b8cbfdd4015a859820228ee8e10 2024-12-07T18:15:29,776 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/98946b8cbfdd4015a859820228ee8e10, entries=150, sequenceid=265, filesize=30.5 K 2024-12-07T18:15:29,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/490091c4fe4845da88dfe1cd3bc3c264 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/490091c4fe4845da88dfe1cd3bc3c264 2024-12-07T18:15:29,780 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/490091c4fe4845da88dfe1cd3bc3c264, entries=150, sequenceid=265, filesize=12.0 K 2024-12-07T18:15:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/a2d40686099e4de9bb9226b8f8bb08b7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a2d40686099e4de9bb9226b8f8bb08b7 2024-12-07T18:15:29,787 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a2d40686099e4de9bb9226b8f8bb08b7, entries=150, sequenceid=265, filesize=12.0 K 2024-12-07T18:15:29,788 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6f3992ce41b26fed1c9fca93d87e0364 in 1274ms, sequenceid=265, compaction requested=true 2024-12-07T18:15:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:29,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-07T18:15:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-07T18:15:29,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-07T18:15:29,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4280 sec 2024-12-07T18:15:29,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4320 sec 2024-12-07T18:15:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:29,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T18:15:29,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:29,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:29,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:29,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:29,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:29,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:29,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076793363a521f406696038c33e4ea8b6a_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595328820/Put/seqid=0 2024-12-07T18:15:29,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742242_1418 (size=14994) 2024-12-07T18:15:29,857 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:29,860 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412076793363a521f406696038c33e4ea8b6a_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076793363a521f406696038c33e4ea8b6a_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:29,861 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/1e56f5c77f2847bcb394a00147640177, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:29,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/1e56f5c77f2847bcb394a00147640177 is 175, key is test_row_0/A:col10/1733595328820/Put/seqid=0 2024-12-07T18:15:29,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742243_1419 (size=39949) 2024-12-07T18:15:30,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595389988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595329911, max timestamp=1733595330000Num uniques 859; 2024-12-07T18:15:30,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:30,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595329911, max timestamp=1733595330000Num uniques 859; 2024-12-07T18:15:30,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:30,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595329911, max timestamp=1733595330000Num uniques 859; 2024-12-07T18:15:30,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:30,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595390046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595390056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595390057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595390057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595390168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595390188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595390189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595390189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,265 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/1e56f5c77f2847bcb394a00147640177 2024-12-07T18:15:30,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/48f1c516ac3245e38b3ae0ee6084f512 is 50, key is test_row_0/B:col10/1733595328820/Put/seqid=0 2024-12-07T18:15:30,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742244_1420 (size=12301) 2024-12-07T18:15:30,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595390388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595390409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595390409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595390409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-07T18:15:30,508 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:30,508 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-07T18:15:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T18:15:30,510 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:30,511 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:30,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T18:15:30,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-07T18:15:30,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:30,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:30,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:30,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:30,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/48f1c516ac3245e38b3ae0ee6084f512 2024-12-07T18:15:30,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/786d31e66d8040c6a375c29c93bcc913 is 50, key is test_row_0/C:col10/1733595328820/Put/seqid=0 2024-12-07T18:15:30,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742245_1421 (size=12301) 2024-12-07T18:15:30,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595390708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595390718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595390728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595390728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:30,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:30,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-07T18:15:30,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:30,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:30,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:30,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T18:15:30,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-07T18:15:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:30,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:30,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:30,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:31,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595391018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,031 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/786d31e66d8040c6a375c29c93bcc913 2024-12-07T18:15:31,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/1e56f5c77f2847bcb394a00147640177 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1e56f5c77f2847bcb394a00147640177 2024-12-07T18:15:31,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1e56f5c77f2847bcb394a00147640177, entries=200, sequenceid=276, filesize=39.0 K 2024-12-07T18:15:31,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/48f1c516ac3245e38b3ae0ee6084f512 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/48f1c516ac3245e38b3ae0ee6084f512 2024-12-07T18:15:31,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/48f1c516ac3245e38b3ae0ee6084f512, entries=150, sequenceid=276, filesize=12.0 K 2024-12-07T18:15:31,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/786d31e66d8040c6a375c29c93bcc913 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/786d31e66d8040c6a375c29c93bcc913 2024-12-07T18:15:31,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/786d31e66d8040c6a375c29c93bcc913, entries=150, sequenceid=276, filesize=12.0 K 2024-12-07T18:15:31,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 6f3992ce41b26fed1c9fca93d87e0364 in 1258ms, sequenceid=276, compaction requested=true 2024-12-07T18:15:31,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:31,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:31,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:31,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:31,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:31,101 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-07T18:15:31,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:31,101 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-07T18:15:31,101 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:31,103 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61465 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-07T18:15:31,103 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 173573 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-07T18:15:31,103 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/B is initiating minor compaction (all files) 2024-12-07T18:15:31,103 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/A is initiating minor compaction (all files) 2024-12-07T18:15:31,103 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/B in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:31,103 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/A in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:31,103 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/45ef4496ba6f43cda6cce9aecbae66f0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/4b4230c9e50b49b0bf412fac571e1a59, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/89a2a6d5aa494de6a12496850578654d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/490091c4fe4845da88dfe1cd3bc3c264, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/48f1c516ac3245e38b3ae0ee6084f512] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=60.0 K 2024-12-07T18:15:31,103 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9f276732d2154c539be64b4b2ccdfe53, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/12796069dd494c82a5556d3f73f1c0a5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/98946b8cbfdd4015a859820228ee8e10, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1e56f5c77f2847bcb394a00147640177] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=169.5 K 2024-12-07T18:15:31,103 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:31,103 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9f276732d2154c539be64b4b2ccdfe53, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/12796069dd494c82a5556d3f73f1c0a5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/98946b8cbfdd4015a859820228ee8e10, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1e56f5c77f2847bcb394a00147640177] 2024-12-07T18:15:31,103 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 45ef4496ba6f43cda6cce9aecbae66f0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733595324377 2024-12-07T18:15:31,104 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f276732d2154c539be64b4b2ccdfe53, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733595324377 2024-12-07T18:15:31,104 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4230c9e50b49b0bf412fac571e1a59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733595325643 2024-12-07T18:15:31,104 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc1a7b11f69a4e6f8d03e8e9ddf64adc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733595325643 2024-12-07T18:15:31,104 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 89a2a6d5aa494de6a12496850578654d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733595326884 2024-12-07T18:15:31,104 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12796069dd494c82a5556d3f73f1c0a5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733595326884 2024-12-07T18:15:31,104 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 490091c4fe4845da88dfe1cd3bc3c264, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733595327610 2024-12-07T18:15:31,105 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98946b8cbfdd4015a859820228ee8e10, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733595327610 2024-12-07T18:15:31,105 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 48f1c516ac3245e38b3ae0ee6084f512, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733595328808 2024-12-07T18:15:31,105 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e56f5c77f2847bcb394a00147640177, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733595328795 2024-12-07T18:15:31,113 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:31,117 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207a56cad8f091b467f847f6002e6790940_6f3992ce41b26fed1c9fca93d87e0364 store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:31,119 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207a56cad8f091b467f847f6002e6790940_6f3992ce41b26fed1c9fca93d87e0364, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:31,120 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a56cad8f091b467f847f6002e6790940_6f3992ce41b26fed1c9fca93d87e0364 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:31,120 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#B#compaction#355 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:31,121 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/5baa7f24d11e4d86929dc6a6403ccc50 is 50, key is test_row_0/B:col10/1733595328820/Put/seqid=0 2024-12-07T18:15:31,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-07T18:15:31,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:31,122 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-07T18:15:31,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:31,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:31,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:31,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:31,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:31,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:31,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742246_1422 (size=4469) 2024-12-07T18:15:31,127 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#A#compaction#354 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:31,128 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/7084fbde111d4932ae547d54d9d904d5 is 175, key is test_row_0/A:col10/1733595328820/Put/seqid=0 2024-12-07T18:15:31,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e6d2346965b74e8ea6c240de7a764a1d_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595330056/Put/seqid=0 2024-12-07T18:15:31,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742247_1423 (size=31835) 2024-12-07T18:15:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T18:15:31,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742248_1424 (size=12454) 2024-12-07T18:15:31,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:31,150 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e6d2346965b74e8ea6c240de7a764a1d_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e6d2346965b74e8ea6c240de7a764a1d_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:31,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742249_1425 (size=12881) 2024-12-07T18:15:31,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2b1da733fad54bf9b1c76c58fcabb1d5, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:31,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2b1da733fad54bf9b1c76c58fcabb1d5 is 175, key is test_row_0/A:col10/1733595330056/Put/seqid=0 2024-12-07T18:15:31,159 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/5baa7f24d11e4d86929dc6a6403ccc50 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/5baa7f24d11e4d86929dc6a6403ccc50 2024-12-07T18:15:31,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742250_1426 (size=31255) 2024-12-07T18:15:31,163 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/B of 6f3992ce41b26fed1c9fca93d87e0364 into 5baa7f24d11e4d86929dc6a6403ccc50(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:31,163 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:31,163 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/B, priority=11, startTime=1733595331101; duration=0sec 2024-12-07T18:15:31,164 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:31,164 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:B 2024-12-07T18:15:31,164 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-07T18:15:31,166 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61465 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-07T18:15:31,166 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/C is initiating minor compaction (all files) 2024-12-07T18:15:31,166 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/C in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:31,166 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/b53761101050478884c911d71d76d848, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/ef31241a21e14b198de2358e9a7e890b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/402316b596fe4365b68836ebe736c550, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a2d40686099e4de9bb9226b8f8bb08b7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/786d31e66d8040c6a375c29c93bcc913] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=60.0 K 2024-12-07T18:15:31,166 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting b53761101050478884c911d71d76d848, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1733595324377 2024-12-07T18:15:31,167 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ef31241a21e14b198de2358e9a7e890b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733595325643 2024-12-07T18:15:31,167 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 402316b596fe4365b68836ebe736c550, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733595326884 2024-12-07T18:15:31,167 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d40686099e4de9bb9226b8f8bb08b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733595327610 2024-12-07T18:15:31,167 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 786d31e66d8040c6a375c29c93bcc913, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733595328808 2024-12-07T18:15:31,175 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#C#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:31,175 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/36479405c3154b45882e25f7c91c5dcb is 50, key is test_row_0/C:col10/1733595328820/Put/seqid=0 2024-12-07T18:15:31,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742251_1427 (size=12881) 2024-12-07T18:15:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:31,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:31,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595391260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595391261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595391262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595391267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595391378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595391378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595391378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595391378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,539 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/7084fbde111d4932ae547d54d9d904d5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7084fbde111d4932ae547d54d9d904d5 2024-12-07T18:15:31,545 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/A of 6f3992ce41b26fed1c9fca93d87e0364 into 7084fbde111d4932ae547d54d9d904d5(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:31,545 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:31,545 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/A, priority=11, startTime=1733595331101; duration=0sec 2024-12-07T18:15:31,545 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:31,545 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:A 2024-12-07T18:15:31,564 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2b1da733fad54bf9b1c76c58fcabb1d5 2024-12-07T18:15:31,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/40d66bff080a4f32a441fef8dc85eb25 is 50, key is test_row_0/B:col10/1733595330056/Put/seqid=0 2024-12-07T18:15:31,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742252_1428 (size=12301) 2024-12-07T18:15:31,586 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/36479405c3154b45882e25f7c91c5dcb as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/36479405c3154b45882e25f7c91c5dcb 2024-12-07T18:15:31,592 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/C of 6f3992ce41b26fed1c9fca93d87e0364 into 36479405c3154b45882e25f7c91c5dcb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:31,592 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:31,592 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/C, priority=11, startTime=1733595331101; duration=0sec 2024-12-07T18:15:31,593 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:31,593 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:C 2024-12-07T18:15:31,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595391598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595391599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595391599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595391599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T18:15:31,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595391918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595391918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595391919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595391919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:31,979 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/40d66bff080a4f32a441fef8dc85eb25 2024-12-07T18:15:31,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/df26db9bd3344ff79d8cd255917e5ae1 is 50, key is test_row_0/C:col10/1733595330056/Put/seqid=0 2024-12-07T18:15:31,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742253_1429 (size=12301) 2024-12-07T18:15:32,390 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/df26db9bd3344ff79d8cd255917e5ae1 2024-12-07T18:15:32,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2b1da733fad54bf9b1c76c58fcabb1d5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2b1da733fad54bf9b1c76c58fcabb1d5 2024-12-07T18:15:32,399 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2b1da733fad54bf9b1c76c58fcabb1d5, entries=150, sequenceid=312, filesize=30.5 K 2024-12-07T18:15:32,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/40d66bff080a4f32a441fef8dc85eb25 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/40d66bff080a4f32a441fef8dc85eb25 2024-12-07T18:15:32,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,407 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/40d66bff080a4f32a441fef8dc85eb25, entries=150, sequenceid=312, filesize=12.0 K 2024-12-07T18:15:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/df26db9bd3344ff79d8cd255917e5ae1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/df26db9bd3344ff79d8cd255917e5ae1 2024-12-07T18:15:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,412 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/df26db9bd3344ff79d8cd255917e5ae1, entries=150, sequenceid=312, filesize=12.0 K 2024-12-07T18:15:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,413 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6f3992ce41b26fed1c9fca93d87e0364 in 1291ms, sequenceid=312, compaction requested=false 2024-12-07T18:15:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:32,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-07T18:15:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-07T18:15:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-07T18:15:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9040 sec 2024-12-07T18:15:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.9110 sec 2024-12-07T18:15:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:32,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T18:15:32,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:32,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:32,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:32,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:32,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073086602e600f450bb9573b2498ad4c08_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595332447/Put/seqid=0 2024-12-07T18:15:32,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742254_1430 (size=12454) 2024-12-07T18:15:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,491 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073086602e600f450bb9573b2498ad4c08_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073086602e600f450bb9573b2498ad4c08_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:32,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,492 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/e259a71f6df84b2f8485c9eadf987e97, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:32,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/e259a71f6df84b2f8485c9eadf987e97 is 175, key is test_row_0/A:col10/1733595332447/Put/seqid=0 2024-12-07T18:15:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742255_1431 (size=31251) 2024-12-07T18:15:32,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,511 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/e259a71f6df84b2f8485c9eadf987e97 2024-12-07T18:15:32,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/8c26e4d074c74fe09a8e21d857eeefb0 is 50, key is test_row_0/B:col10/1733595332447/Put/seqid=0 2024-12-07T18:15:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:32,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742256_1432 (size=9857) 2024-12-07T18:15:32,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595392539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595332534, max timestamp=1733595332550Num uniques 859; 2024-12-07T18:15:32,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595332534, max timestamp=1733595332550Num uniques 859; 2024-12-07T18:15:32,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:15:32,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:15:32,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595332534, max timestamp=1733595332550Num uniques 859; 2024-12-07T18:15:32,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:15:32,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595392592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595392593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595392602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-07T18:15:32,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595392658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,668 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,669 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-07T18:15:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-07T18:15:32,671 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:32,671 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:32,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:32,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595392708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595392718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595392718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-07T18:15:32,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-07T18:15:32,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:32,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:32,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:32,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:32,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:32,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595392878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595392928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/8c26e4d074c74fe09a8e21d857eeefb0 2024-12-07T18:15:32,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595392938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595392938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:32,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:32,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/7494e03440224e57a2e66ee8538d1d2d is 50, key is test_row_0/C:col10/1733595332447/Put/seqid=0 2024-12-07T18:15:32,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742257_1433 (size=9857) 2024-12-07T18:15:32,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/7494e03440224e57a2e66ee8538d1d2d 2024-12-07T18:15:32,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/e259a71f6df84b2f8485c9eadf987e97 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e259a71f6df84b2f8485c9eadf987e97 2024-12-07T18:15:32,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e259a71f6df84b2f8485c9eadf987e97, entries=150, sequenceid=326, filesize=30.5 K 2024-12-07T18:15:32,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/8c26e4d074c74fe09a8e21d857eeefb0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8c26e4d074c74fe09a8e21d857eeefb0 2024-12-07T18:15:32,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-07T18:15:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:32,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:32,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8c26e4d074c74fe09a8e21d857eeefb0, entries=100, sequenceid=326, filesize=9.6 K 2024-12-07T18:15:32,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/7494e03440224e57a2e66ee8538d1d2d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/7494e03440224e57a2e66ee8538d1d2d 2024-12-07T18:15:32,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/7494e03440224e57a2e66ee8538d1d2d, entries=100, sequenceid=326, filesize=9.6 K 2024-12-07T18:15:32,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6f3992ce41b26fed1c9fca93d87e0364 in 537ms, sequenceid=326, compaction requested=true 2024-12-07T18:15:32,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:32,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:32,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:32,985 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:32,985 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:32,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:32,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:32,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:32,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:32,986 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:32,986 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/B is initiating minor compaction (all files) 2024-12-07T18:15:32,986 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/B in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:32,986 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/5baa7f24d11e4d86929dc6a6403ccc50, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/40d66bff080a4f32a441fef8dc85eb25, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8c26e4d074c74fe09a8e21d857eeefb0] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=34.2 K 2024-12-07T18:15:32,987 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94341 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:32,987 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/A is initiating minor compaction (all files) 2024-12-07T18:15:32,987 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/A in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:32,987 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7084fbde111d4932ae547d54d9d904d5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2b1da733fad54bf9b1c76c58fcabb1d5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e259a71f6df84b2f8485c9eadf987e97] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=92.1 K 2024-12-07T18:15:32,987 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 5baa7f24d11e4d86929dc6a6403ccc50, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733595328808 2024-12-07T18:15:32,987 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:32,987 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7084fbde111d4932ae547d54d9d904d5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2b1da733fad54bf9b1c76c58fcabb1d5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e259a71f6df84b2f8485c9eadf987e97] 2024-12-07T18:15:32,987 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 40d66bff080a4f32a441fef8dc85eb25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733595330044 2024-12-07T18:15:32,987 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7084fbde111d4932ae547d54d9d904d5, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733595328808 2024-12-07T18:15:32,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-07T18:15:32,988 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c26e4d074c74fe09a8e21d857eeefb0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733595331260 2024-12-07T18:15:32,988 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b1da733fad54bf9b1c76c58fcabb1d5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733595330044 2024-12-07T18:15:32,989 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting e259a71f6df84b2f8485c9eadf987e97, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733595331260 2024-12-07T18:15:32,999 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:32,999 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/08fc1133d9ad47c2b79b2237aadd61e7 is 50, key is test_row_0/B:col10/1733595332447/Put/seqid=0 2024-12-07T18:15:33,007 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:33,009 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412077919994d96a34aa29b28f67ac778576e_6f3992ce41b26fed1c9fca93d87e0364 store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:33,011 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412077919994d96a34aa29b28f67ac778576e_6f3992ce41b26fed1c9fca93d87e0364, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:33,011 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412077919994d96a34aa29b28f67ac778576e_6f3992ce41b26fed1c9fca93d87e0364 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:33,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742258_1434 (size=12983) 2024-12-07T18:15:33,024 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/08fc1133d9ad47c2b79b2237aadd61e7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/08fc1133d9ad47c2b79b2237aadd61e7 2024-12-07T18:15:33,031 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/B of 6f3992ce41b26fed1c9fca93d87e0364 into 08fc1133d9ad47c2b79b2237aadd61e7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:33,031 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:33,031 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/B, priority=13, startTime=1733595332985; duration=0sec 2024-12-07T18:15:33,032 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:33,032 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:B 2024-12-07T18:15:33,032 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:33,033 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:33,033 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/C is initiating minor compaction (all files) 2024-12-07T18:15:33,033 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/C in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,033 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/36479405c3154b45882e25f7c91c5dcb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/df26db9bd3344ff79d8cd255917e5ae1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/7494e03440224e57a2e66ee8538d1d2d] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=34.2 K 2024-12-07T18:15:33,034 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 36479405c3154b45882e25f7c91c5dcb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733595328808 2024-12-07T18:15:33,034 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting df26db9bd3344ff79d8cd255917e5ae1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733595330044 2024-12-07T18:15:33,035 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7494e03440224e57a2e66ee8538d1d2d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733595331260 2024-12-07T18:15:33,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742259_1435 (size=4469) 2024-12-07T18:15:33,042 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#A#compaction#364 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:33,043 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/66518c3a7d6647e3a26d080795ee37b7 is 175, key is test_row_0/A:col10/1733595332447/Put/seqid=0 2024-12-07T18:15:33,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-07T18:15:33,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:33,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:33,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:33,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:33,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:33,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:33,062 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#C#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:33,063 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6431421333714e3aa5fe961895a542b6 is 50, key is test_row_0/C:col10/1733595332447/Put/seqid=0 2024-12-07T18:15:33,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120769e57043e566430bb67326287ba2e825_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595333055/Put/seqid=0 2024-12-07T18:15:33,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-07T18:15:33,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:33,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742260_1436 (size=32044) 2024-12-07T18:15:33,142 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/66518c3a7d6647e3a26d080795ee37b7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/66518c3a7d6647e3a26d080795ee37b7 2024-12-07T18:15:33,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742261_1437 (size=12983) 2024-12-07T18:15:33,147 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/A of 6f3992ce41b26fed1c9fca93d87e0364 into 66518c3a7d6647e3a26d080795ee37b7(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:33,147 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:33,148 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/A, priority=13, startTime=1733595332985; duration=0sec 2024-12-07T18:15:33,148 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:33,148 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:A 2024-12-07T18:15:33,152 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6431421333714e3aa5fe961895a542b6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6431421333714e3aa5fe961895a542b6 2024-12-07T18:15:33,160 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/C of 6f3992ce41b26fed1c9fca93d87e0364 into 6431421333714e3aa5fe961895a542b6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:33,160 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:33,160 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/C, priority=13, startTime=1733595332985; duration=0sec 2024-12-07T18:15:33,160 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:33,160 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:C 2024-12-07T18:15:33,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742262_1438 (size=14994) 2024-12-07T18:15:33,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595393192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595393198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595393248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595393258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595393258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-07T18:15:33,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-07T18:15:33,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595393319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-07T18:15:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595393538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,568 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:33,571 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120769e57043e566430bb67326287ba2e825_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120769e57043e566430bb67326287ba2e825_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:33,572 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/f72860aeedf441a99c65a2f8c6110dc2, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:33,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/f72860aeedf441a99c65a2f8c6110dc2 is 175, key is test_row_0/A:col10/1733595333055/Put/seqid=0 2024-12-07T18:15:33,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742263_1439 (size=39949) 2024-12-07T18:15:33,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-07T18:15:33,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:33,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595393728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-07T18:15:33,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:33,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,744 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595393768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595393779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595393779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-07T18:15:33,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595393859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:33,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:33,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:33,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-07T18:15:33,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:33,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:33,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:33,976 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=362, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/f72860aeedf441a99c65a2f8c6110dc2 2024-12-07T18:15:33,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/9da5715465af4249ab20fc43f934b929 is 50, key is test_row_0/B:col10/1733595333055/Put/seqid=0 2024-12-07T18:15:33,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742264_1440 (size=12301) 2024-12-07T18:15:33,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/9da5715465af4249ab20fc43f934b929 2024-12-07T18:15:33,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6e652088c519459ba37956d3187e3b96 is 50, key is test_row_0/C:col10/1733595333055/Put/seqid=0 2024-12-07T18:15:34,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742265_1441 (size=12301) 2024-12-07T18:15:34,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6e652088c519459ba37956d3187e3b96 2024-12-07T18:15:34,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/f72860aeedf441a99c65a2f8c6110dc2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/f72860aeedf441a99c65a2f8c6110dc2 2024-12-07T18:15:34,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/f72860aeedf441a99c65a2f8c6110dc2, entries=200, sequenceid=362, filesize=39.0 K 2024-12-07T18:15:34,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/9da5715465af4249ab20fc43f934b929 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/9da5715465af4249ab20fc43f934b929 2024-12-07T18:15:34,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/9da5715465af4249ab20fc43f934b929, entries=150, sequenceid=362, filesize=12.0 K 2024-12-07T18:15:34,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/6e652088c519459ba37956d3187e3b96 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e652088c519459ba37956d3187e3b96 2024-12-07T18:15:34,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e652088c519459ba37956d3187e3b96, entries=150, sequenceid=362, filesize=12.0 K 2024-12-07T18:15:34,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 6f3992ce41b26fed1c9fca93d87e0364 in 965ms, sequenceid=362, compaction requested=false 2024-12-07T18:15:34,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:34,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-07T18:15:34,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:34,050 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:15:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:34,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207c7087d43c19a4ed4b3b9f7406de57ce6_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595333190/Put/seqid=0 2024-12-07T18:15:34,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742266_1442 (size=12454) 2024-12-07T18:15:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:34,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. as already flushing 2024-12-07T18:15:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:34,465 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207c7087d43c19a4ed4b3b9f7406de57ce6_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c7087d43c19a4ed4b3b9f7406de57ce6_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:34,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2cdbe4e934604764b0dd3f62fd4c78de, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:34,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2cdbe4e934604764b0dd3f62fd4c78de is 175, key is test_row_0/A:col10/1733595333190/Put/seqid=0 2024-12-07T18:15:34,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742267_1443 (size=31255) 2024-12-07T18:15:34,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595334430, max timestamp=1733595334506Num uniques 859; 2024-12-07T18:15:34,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595334430, max timestamp=1733595334506Num uniques 859; 2024-12-07T18:15:34,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595334430, max timestamp=1733595334506Num uniques 859; 2024-12-07T18:15:34,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:34,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:34,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:15:34,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595394665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:34,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42686 deadline: 1733595394748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:34,761 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:34,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595394778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:34,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42634 deadline: 1733595394788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:34,802 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42642 deadline: 1733595394798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42702 deadline: 1733595394798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:34,811 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:34,811 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 6f3992ce41b26fed1c9fca93d87e0364 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-07T18:15:34,870 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2cdbe4e934604764b0dd3f62fd4c78de 2024-12-07T18:15:34,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/7824080818334d65a5520a09b4124c5d is 50, key is test_row_0/B:col10/1733595333190/Put/seqid=0 2024-12-07T18:15:34,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742268_1444 (size=12301) 2024-12-07T18:15:34,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595394988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:35,281 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/7824080818334d65a5520a09b4124c5d 2024-12-07T18:15:35,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/3f2a2b28327543b58d5f145b5bb0aebc is 50, key is test_row_0/C:col10/1733595333190/Put/seqid=0 2024-12-07T18:15:35,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742269_1445 (size=12301) 2024-12-07T18:15:35,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595395308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:35,435 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T18:15:35,691 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/3f2a2b28327543b58d5f145b5bb0aebc 2024-12-07T18:15:35,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2cdbe4e934604764b0dd3f62fd4c78de as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2cdbe4e934604764b0dd3f62fd4c78de 2024-12-07T18:15:35,713 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2cdbe4e934604764b0dd3f62fd4c78de, entries=150, sequenceid=374, filesize=30.5 K 2024-12-07T18:15:35,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/7824080818334d65a5520a09b4124c5d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/7824080818334d65a5520a09b4124c5d 2024-12-07T18:15:35,735 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/7824080818334d65a5520a09b4124c5d, entries=150, sequenceid=374, filesize=12.0 K 2024-12-07T18:15:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/3f2a2b28327543b58d5f145b5bb0aebc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3f2a2b28327543b58d5f145b5bb0aebc 2024-12-07T18:15:35,741 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3f2a2b28327543b58d5f145b5bb0aebc, entries=150, sequenceid=374, filesize=12.0 K 2024-12-07T18:15:35,743 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6f3992ce41b26fed1c9fca93d87e0364 in 1693ms, sequenceid=374, compaction requested=true 2024-12-07T18:15:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-07T18:15:35,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-07T18:15:35,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-07T18:15:35,747 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0740 sec 2024-12-07T18:15:35,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 3.0780 sec 2024-12-07T18:15:35,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:35,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-07T18:15:35,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:35,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:35,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:35,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:35,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:35,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:35,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e7ac84eb0af94e0b92bf965aa85b4eb9_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595334663/Put/seqid=0 2024-12-07T18:15:35,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742270_1446 (size=14994) 2024-12-07T18:15:35,872 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:35,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e7ac84eb0af94e0b92bf965aa85b4eb9_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e7ac84eb0af94e0b92bf965aa85b4eb9_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:35,879 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/36781334cbf74dc4add0b95d04a47b12, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:35,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/36781334cbf74dc4add0b95d04a47b12 is 175, key is test_row_0/A:col10/1733595334663/Put/seqid=0 2024-12-07T18:15:35,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742271_1447 (size=39949) 2024-12-07T18:15:35,892 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=411, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/36781334cbf74dc4add0b95d04a47b12 2024-12-07T18:15:35,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/191d1395b96049c3ba3fe3057b423f78 is 50, key is test_row_0/B:col10/1733595334663/Put/seqid=0 2024-12-07T18:15:35,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742272_1448 (size=12301) 2024-12-07T18:15:35,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/191d1395b96049c3ba3fe3057b423f78 2024-12-07T18:15:35,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42646 deadline: 1733595395926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:35,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:35,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:35,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:15:35,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/a29cff4fc9944b6da398f66149323829 is 50, key is test_row_0/C:col10/1733595334663/Put/seqid=0 2024-12-07T18:15:35,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742273_1449 (size=12301) 2024-12-07T18:15:35,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/a29cff4fc9944b6da398f66149323829 2024-12-07T18:15:35,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/36781334cbf74dc4add0b95d04a47b12 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/36781334cbf74dc4add0b95d04a47b12 2024-12-07T18:15:35,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/36781334cbf74dc4add0b95d04a47b12, entries=200, sequenceid=411, filesize=39.0 K 2024-12-07T18:15:35,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/191d1395b96049c3ba3fe3057b423f78 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/191d1395b96049c3ba3fe3057b423f78 2024-12-07T18:15:35,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/191d1395b96049c3ba3fe3057b423f78, entries=150, sequenceid=411, filesize=12.0 K 2024-12-07T18:15:35,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/a29cff4fc9944b6da398f66149323829 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a29cff4fc9944b6da398f66149323829 2024-12-07T18:15:35,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a29cff4fc9944b6da398f66149323829, entries=150, sequenceid=411, filesize=12.0 K 2024-12-07T18:15:35,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 6f3992ce41b26fed1c9fca93d87e0364 in 143ms, sequenceid=411, compaction requested=true 2024-12-07T18:15:35,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:35,989 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:15:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:35,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:35,990 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143197 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:15:35,990 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/A is initiating minor compaction (all files) 2024-12-07T18:15:35,990 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/A in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:35,990 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/66518c3a7d6647e3a26d080795ee37b7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/f72860aeedf441a99c65a2f8c6110dc2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2cdbe4e934604764b0dd3f62fd4c78de, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/36781334cbf74dc4add0b95d04a47b12] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=139.8 K 2024-12-07T18:15:35,990 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:35,990 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/66518c3a7d6647e3a26d080795ee37b7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/f72860aeedf441a99c65a2f8c6110dc2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2cdbe4e934604764b0dd3f62fd4c78de, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/36781334cbf74dc4add0b95d04a47b12] 2024-12-07T18:15:35,991 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:15:35,991 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66518c3a7d6647e3a26d080795ee37b7, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733595330055 2024-12-07T18:15:35,991 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting f72860aeedf441a99c65a2f8c6110dc2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733595332590 2024-12-07T18:15:35,992 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cdbe4e934604764b0dd3f62fd4c78de, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733595333145 2024-12-07T18:15:35,992 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:15:35,992 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/B is initiating minor compaction (all files) 2024-12-07T18:15:35,992 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/B in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:35,992 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/08fc1133d9ad47c2b79b2237aadd61e7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/9da5715465af4249ab20fc43f934b929, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/7824080818334d65a5520a09b4124c5d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/191d1395b96049c3ba3fe3057b423f78] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=48.7 K 2024-12-07T18:15:35,992 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36781334cbf74dc4add0b95d04a47b12, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733595334601 2024-12-07T18:15:35,993 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 08fc1133d9ad47c2b79b2237aadd61e7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733595330055 2024-12-07T18:15:35,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:35,993 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da5715465af4249ab20fc43f934b929, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733595332590 2024-12-07T18:15:35,993 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7824080818334d65a5520a09b4124c5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733595333145 2024-12-07T18:15:35,994 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 191d1395b96049c3ba3fe3057b423f78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733595334648 2024-12-07T18:15:36,001 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:36,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6f3992ce41b26fed1c9fca93d87e0364:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:36,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:36,012 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:36,023 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#B#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:36,024 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/7a1b07bc736e448585ee9aff2a031f54 is 50, key is test_row_0/B:col10/1733595334663/Put/seqid=0 2024-12-07T18:15:36,041 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207b1c782b839d4420380da967c289321e8_6f3992ce41b26fed1c9fca93d87e0364 store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:36,045 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207b1c782b839d4420380da967c289321e8_6f3992ce41b26fed1c9fca93d87e0364, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:36,045 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207b1c782b839d4420380da967c289321e8_6f3992ce41b26fed1c9fca93d87e0364 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:36,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:36,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:15:36,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:36,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:36,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:36,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:36,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:36,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:36,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742274_1450 (size=13119) 2024-12-07T18:15:36,114 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/7a1b07bc736e448585ee9aff2a031f54 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/7a1b07bc736e448585ee9aff2a031f54 2024-12-07T18:15:36,119 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/B of 6f3992ce41b26fed1c9fca93d87e0364 into 7a1b07bc736e448585ee9aff2a031f54(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:36,119 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:36,119 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/B, priority=12, startTime=1733595335991; duration=0sec 2024-12-07T18:15:36,119 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:36,119 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:B 2024-12-07T18:15:36,119 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:15:36,120 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:15:36,121 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 6f3992ce41b26fed1c9fca93d87e0364/C is initiating minor compaction (all files) 2024-12-07T18:15:36,121 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6f3992ce41b26fed1c9fca93d87e0364/C in TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:36,121 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6431421333714e3aa5fe961895a542b6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e652088c519459ba37956d3187e3b96, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3f2a2b28327543b58d5f145b5bb0aebc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a29cff4fc9944b6da398f66149323829] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp, totalSize=48.7 K 2024-12-07T18:15:36,121 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6431421333714e3aa5fe961895a542b6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733595330055 2024-12-07T18:15:36,121 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e652088c519459ba37956d3187e3b96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733595332590 2024-12-07T18:15:36,122 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f2a2b28327543b58d5f145b5bb0aebc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1733595333145 2024-12-07T18:15:36,122 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting a29cff4fc9944b6da398f66149323829, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1733595334648 2024-12-07T18:15:36,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742275_1451 (size=4469) 2024-12-07T18:15:36,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207849981e3ff684f799b5822091341dc60_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595335876/Put/seqid=0 2024-12-07T18:15:36,167 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#C#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:36,168 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/2a966fd5621a4eef80f5cdf0993ffb3e is 50, key is test_row_0/C:col10/1733595334663/Put/seqid=0 2024-12-07T18:15:36,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742276_1452 (size=14994) 2024-12-07T18:15:36,210 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:36,214 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207849981e3ff684f799b5822091341dc60_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207849981e3ff684f799b5822091341dc60_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:36,215 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2d98659356d04bd180ed9078ab603c91, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:36,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2d98659356d04bd180ed9078ab603c91 is 175, key is test_row_0/A:col10/1733595335876/Put/seqid=0 2024-12-07T18:15:36,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742277_1453 (size=13119) 2024-12-07T18:15:36,235 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/2a966fd5621a4eef80f5cdf0993ffb3e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/2a966fd5621a4eef80f5cdf0993ffb3e 2024-12-07T18:15:36,245 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/C of 6f3992ce41b26fed1c9fca93d87e0364 into 2a966fd5621a4eef80f5cdf0993ffb3e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:36,245 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:36,245 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/C, priority=12, startTime=1733595336001; duration=0sec 2024-12-07T18:15:36,245 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:36,245 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:C 2024-12-07T18:15:36,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742278_1454 (size=39949) 2024-12-07T18:15:36,257 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2d98659356d04bd180ed9078ab603c91 2024-12-07T18:15:36,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/22369dac6a3a486880d831e881a0feef is 50, key is test_row_0/B:col10/1733595335876/Put/seqid=0 2024-12-07T18:15:36,297 INFO [Thread-1621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1621. 2024-12-07T18:15:36,297 DEBUG [Thread-1621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:36,297 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,297 DEBUG [Thread-1621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,297 INFO [Thread-1617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1617. 2024-12-07T18:15:36,297 DEBUG [Thread-1617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:36,297 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,297 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,298 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,304 INFO [Thread-1602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1602. 2024-12-07T18:15:36,304 DEBUG [Thread-1602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:36,304 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,304 INFO [Thread-1613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1613. 2024-12-07T18:15:36,304 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,305 DEBUG [Thread-1613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:36,305 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,305 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,314 INFO [Thread-1615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1615. 2024-12-07T18:15:36,314 DEBUG [Thread-1615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:36,314 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,314 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,320 INFO [Thread-1619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1619. 2024-12-07T18:15:36,320 DEBUG [Thread-1619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:36,320 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,320 DEBUG [Thread-1619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742279_1455 (size=12301) 2024-12-07T18:15:36,527 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6f3992ce41b26fed1c9fca93d87e0364#A#compaction#375 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:36,528 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/1a5f430919cb4d51843f3d12d3706bbe is 175, key is test_row_0/A:col10/1733595334663/Put/seqid=0 2024-12-07T18:15:36,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742280_1456 (size=32073) 2024-12-07T18:15:36,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/22369dac6a3a486880d831e881a0feef 2024-12-07T18:15:36,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/da572fd36de145c4950056a224b64ce3 is 50, key is test_row_0/C:col10/1733595335876/Put/seqid=0 2024-12-07T18:15:36,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742281_1457 (size=12301) 2024-12-07T18:15:36,770 INFO [Thread-1604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1604. 2024-12-07T18:15:36,770 DEBUG [Thread-1604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:36,770 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,770 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,821 INFO [Thread-1606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1606. 2024-12-07T18:15:36,821 DEBUG [Thread-1606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:36,821 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,822 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,830 INFO [Thread-1610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1610. 2024-12-07T18:15:36,831 DEBUG [Thread-1610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:36,831 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,831 DEBUG [Thread-1610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-07T18:15:36,840 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:36,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733595336768, max timestamp=1733595336840Num uniques 859; 2024-12-07T18:15:36,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733595336768, max timestamp=1733595336840Num uniques 859; 2024-12-07T18:15:36,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733595336768, max timestamp=1733595336840Num uniques 859; 2024-12-07T18:15:36,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-07T18:15:36,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-07T18:15:36,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-07T18:15:36,844 INFO [Thread-1608 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1608. 2024-12-07T18:15:36,845 DEBUG [Thread-1608 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:15:36,845 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,845 DEBUG [Thread-1608 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,845 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 107 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 880 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2637 rows 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 884 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2649 rows 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 883 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2648 rows 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 889 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2666 rows 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 883 2024-12-07T18:15:36,845 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2644 rows 2024-12-07T18:15:36,845 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:15:36,845 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:15:36,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:36,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:15:36,846 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T18:15:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T18:15:36,852 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595336851"}]},"ts":"1733595336851"} 2024-12-07T18:15:36,853 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T18:15:36,853 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T18:15:36,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T18:15:36,856 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, UNASSIGN}] 2024-12-07T18:15:36,857 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, UNASSIGN 2024-12-07T18:15:36,858 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=6f3992ce41b26fed1c9fca93d87e0364, regionState=CLOSING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:36,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, UNASSIGN because future has completed 2024-12-07T18:15:36,860 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T18:15:36,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:15:36,937 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/1a5f430919cb4d51843f3d12d3706bbe as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1a5f430919cb4d51843f3d12d3706bbe 2024-12-07T18:15:36,942 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6f3992ce41b26fed1c9fca93d87e0364/A of 6f3992ce41b26fed1c9fca93d87e0364 into 1a5f430919cb4d51843f3d12d3706bbe(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:36,942 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:36,942 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., storeName=6f3992ce41b26fed1c9fca93d87e0364/A, priority=12, startTime=1733595335989; duration=0sec 2024-12-07T18:15:36,942 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:36,942 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6f3992ce41b26fed1c9fca93d87e0364:A 2024-12-07T18:15:36,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T18:15:37,014 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:37,014 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T18:15:37,014 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 6f3992ce41b26fed1c9fca93d87e0364, disabling compactions & flushes 2024-12-07T18:15:37,014 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:37,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/da572fd36de145c4950056a224b64ce3 2024-12-07T18:15:37,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/2d98659356d04bd180ed9078ab603c91 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2d98659356d04bd180ed9078ab603c91 2024-12-07T18:15:37,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2d98659356d04bd180ed9078ab603c91, entries=200, sequenceid=421, filesize=39.0 K 2024-12-07T18:15:37,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/22369dac6a3a486880d831e881a0feef as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/22369dac6a3a486880d831e881a0feef 2024-12-07T18:15:37,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/22369dac6a3a486880d831e881a0feef, entries=150, sequenceid=421, filesize=12.0 K 2024-12-07T18:15:37,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/da572fd36de145c4950056a224b64ce3 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/da572fd36de145c4950056a224b64ce3 2024-12-07T18:15:37,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/da572fd36de145c4950056a224b64ce3, entries=150, sequenceid=421, filesize=12.0 K 2024-12-07T18:15:37,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=25.65 KB/26265 for 6f3992ce41b26fed1c9fca93d87e0364 in 1111ms, sequenceid=421, compaction requested=false 2024-12-07T18:15:37,161 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:37,161 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:37,162 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. after waiting 1 ms 2024-12-07T18:15:37,162 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:37,162 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 6f3992ce41b26fed1c9fca93d87e0364 3/3 column families, dataSize=25.65 KB heapSize=63.49 KB 2024-12-07T18:15:37,162 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=A 2024-12-07T18:15:37,162 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:37,162 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=B 2024-12-07T18:15:37,162 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:37,162 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6f3992ce41b26fed1c9fca93d87e0364, store=C 2024-12-07T18:15:37,162 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:37,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6f3992ce41b26fed1c9fca93d87e0364: 2024-12-07T18:15:37,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T18:15:37,179 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412072dad870d0ad44aa38c31220fd8997bee_6f3992ce41b26fed1c9fca93d87e0364 is 50, key is test_row_0/A:col10/1733595336768/Put/seqid=0 2024-12-07T18:15:37,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742282_1458 (size=12454) 2024-12-07T18:15:37,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T18:15:37,613 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:37,617 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412072dad870d0ad44aa38c31220fd8997bee_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072dad870d0ad44aa38c31220fd8997bee_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:37,619 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/5f1581fa9250413da3cdf5b24db34541, store: [table=TestAcidGuarantees family=A region=6f3992ce41b26fed1c9fca93d87e0364] 2024-12-07T18:15:37,620 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/5f1581fa9250413da3cdf5b24db34541 is 175, key is test_row_0/A:col10/1733595336768/Put/seqid=0 2024-12-07T18:15:37,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742283_1459 (size=31255) 2024-12-07T18:15:37,625 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=445, memsize=8.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/5f1581fa9250413da3cdf5b24db34541 2024-12-07T18:15:37,632 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/dcff1148526643c59d4ad818c0ac053e is 50, key is test_row_0/B:col10/1733595336768/Put/seqid=0 2024-12-07T18:15:37,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742284_1460 (size=12301) 2024-12-07T18:15:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T18:15:38,039 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/dcff1148526643c59d4ad818c0ac053e 2024-12-07T18:15:38,046 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/f5e324eafa8b4f4fa27bd8591242d8af is 50, key is test_row_0/C:col10/1733595336768/Put/seqid=0 2024-12-07T18:15:38,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742285_1461 (size=12301) 2024-12-07T18:15:38,449 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.55 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/f5e324eafa8b4f4fa27bd8591242d8af 2024-12-07T18:15:38,453 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/A/5f1581fa9250413da3cdf5b24db34541 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/5f1581fa9250413da3cdf5b24db34541 2024-12-07T18:15:38,455 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/5f1581fa9250413da3cdf5b24db34541, entries=150, sequenceid=445, filesize=30.5 K 2024-12-07T18:15:38,456 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/B/dcff1148526643c59d4ad818c0ac053e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/dcff1148526643c59d4ad818c0ac053e 2024-12-07T18:15:38,459 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/dcff1148526643c59d4ad818c0ac053e, entries=150, sequenceid=445, filesize=12.0 K 2024-12-07T18:15:38,459 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/.tmp/C/f5e324eafa8b4f4fa27bd8591242d8af as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/f5e324eafa8b4f4fa27bd8591242d8af 2024-12-07T18:15:38,462 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/f5e324eafa8b4f4fa27bd8591242d8af, entries=150, sequenceid=445, filesize=12.0 K 2024-12-07T18:15:38,463 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~25.65 KB/26265, heapSize ~63.45 KB/64968, currentSize=0 B/0 for 6f3992ce41b26fed1c9fca93d87e0364 in 1301ms, sequenceid=445, compaction requested=true 2024-12-07T18:15:38,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/79df6dcd289848cfa30a22f1e6c8ef42, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1726a760c8494ac4a6d0d54da370a68d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/307b30d7222846e8b612222083058705, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9c83e9c373f448c490c8675945bcff82, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9dbb962ad2fc42b68766266b9bceece8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/17ebcf15ce8549048f806f72acca9607, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a2d3eea46ec14a409c172267bd8e674e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7a180b12a7b24cc4a2e3f788520b853c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a6d81a195735467da1a93e531b4ca472, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e60bde73cc8642b1a3bd64ba7d7a52ad, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/ccdde20de656437daf7555cc88639211, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2a2afdd3fe3b43bc988dea13234fb416, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9f276732d2154c539be64b4b2ccdfe53, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/12796069dd494c82a5556d3f73f1c0a5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/98946b8cbfdd4015a859820228ee8e10, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1e56f5c77f2847bcb394a00147640177, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7084fbde111d4932ae547d54d9d904d5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2b1da733fad54bf9b1c76c58fcabb1d5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/66518c3a7d6647e3a26d080795ee37b7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e259a71f6df84b2f8485c9eadf987e97, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/f72860aeedf441a99c65a2f8c6110dc2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2cdbe4e934604764b0dd3f62fd4c78de, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/36781334cbf74dc4add0b95d04a47b12] to archive 2024-12-07T18:15:38,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:15:38,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/79df6dcd289848cfa30a22f1e6c8ef42 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/79df6dcd289848cfa30a22f1e6c8ef42 2024-12-07T18:15:38,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1726a760c8494ac4a6d0d54da370a68d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1726a760c8494ac4a6d0d54da370a68d 2024-12-07T18:15:38,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/307b30d7222846e8b612222083058705 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/307b30d7222846e8b612222083058705 2024-12-07T18:15:38,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9c83e9c373f448c490c8675945bcff82 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9c83e9c373f448c490c8675945bcff82 2024-12-07T18:15:38,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9dbb962ad2fc42b68766266b9bceece8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9dbb962ad2fc42b68766266b9bceece8 2024-12-07T18:15:38,469 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/17ebcf15ce8549048f806f72acca9607 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/17ebcf15ce8549048f806f72acca9607 2024-12-07T18:15:38,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a2d3eea46ec14a409c172267bd8e674e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a2d3eea46ec14a409c172267bd8e674e 2024-12-07T18:15:38,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7a180b12a7b24cc4a2e3f788520b853c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7a180b12a7b24cc4a2e3f788520b853c 2024-12-07T18:15:38,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a6d81a195735467da1a93e531b4ca472 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/a6d81a195735467da1a93e531b4ca472 2024-12-07T18:15:38,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e60bde73cc8642b1a3bd64ba7d7a52ad to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e60bde73cc8642b1a3bd64ba7d7a52ad 2024-12-07T18:15:38,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/ccdde20de656437daf7555cc88639211 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/ccdde20de656437daf7555cc88639211 2024-12-07T18:15:38,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2a2afdd3fe3b43bc988dea13234fb416 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2a2afdd3fe3b43bc988dea13234fb416 2024-12-07T18:15:38,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9f276732d2154c539be64b4b2ccdfe53 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/9f276732d2154c539be64b4b2ccdfe53 2024-12-07T18:15:38,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/cc1a7b11f69a4e6f8d03e8e9ddf64adc 2024-12-07T18:15:38,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/12796069dd494c82a5556d3f73f1c0a5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/12796069dd494c82a5556d3f73f1c0a5 2024-12-07T18:15:38,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/98946b8cbfdd4015a859820228ee8e10 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/98946b8cbfdd4015a859820228ee8e10 2024-12-07T18:15:38,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1e56f5c77f2847bcb394a00147640177 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1e56f5c77f2847bcb394a00147640177 2024-12-07T18:15:38,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7084fbde111d4932ae547d54d9d904d5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/7084fbde111d4932ae547d54d9d904d5 2024-12-07T18:15:38,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2b1da733fad54bf9b1c76c58fcabb1d5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2b1da733fad54bf9b1c76c58fcabb1d5 2024-12-07T18:15:38,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/66518c3a7d6647e3a26d080795ee37b7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/66518c3a7d6647e3a26d080795ee37b7 2024-12-07T18:15:38,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e259a71f6df84b2f8485c9eadf987e97 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/e259a71f6df84b2f8485c9eadf987e97 2024-12-07T18:15:38,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/f72860aeedf441a99c65a2f8c6110dc2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/f72860aeedf441a99c65a2f8c6110dc2 2024-12-07T18:15:38,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2cdbe4e934604764b0dd3f62fd4c78de to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2cdbe4e934604764b0dd3f62fd4c78de 2024-12-07T18:15:38,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/36781334cbf74dc4add0b95d04a47b12 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/36781334cbf74dc4add0b95d04a47b12 2024-12-07T18:15:38,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/416f684013bf4c659d6fbd78ba1f51b4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8fdd116bc4d142b592da875c7b63b228, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ed39a0177db14705a6705dc309301c2a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/6443af7d57204b4aa07a7f88eb042088, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/0a09572afd9d473eb81a4d6dfd7ea960, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/f404a08b5fa943cb8f79f874a7dfb2c0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/24b33633582f4d04b59a65d85aad9c9d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ea8b82784efa4737b97e19cf531021ab, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/51aa6fd313804281bff7a10c347a9e08, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/71a7573988944da284371daebea23b8c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/b858ec743fe146a1b07810d95369ccc1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/45ef4496ba6f43cda6cce9aecbae66f0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/04466b563e284746b2d23edbd1b88f5b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/4b4230c9e50b49b0bf412fac571e1a59, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/89a2a6d5aa494de6a12496850578654d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/490091c4fe4845da88dfe1cd3bc3c264, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/5baa7f24d11e4d86929dc6a6403ccc50, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/48f1c516ac3245e38b3ae0ee6084f512, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/40d66bff080a4f32a441fef8dc85eb25, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/08fc1133d9ad47c2b79b2237aadd61e7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8c26e4d074c74fe09a8e21d857eeefb0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/9da5715465af4249ab20fc43f934b929, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/7824080818334d65a5520a09b4124c5d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/191d1395b96049c3ba3fe3057b423f78] to archive 2024-12-07T18:15:38,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:15:38,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/416f684013bf4c659d6fbd78ba1f51b4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/416f684013bf4c659d6fbd78ba1f51b4 2024-12-07T18:15:38,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8fdd116bc4d142b592da875c7b63b228 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8fdd116bc4d142b592da875c7b63b228 2024-12-07T18:15:38,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ed39a0177db14705a6705dc309301c2a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ed39a0177db14705a6705dc309301c2a 2024-12-07T18:15:38,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/6443af7d57204b4aa07a7f88eb042088 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/6443af7d57204b4aa07a7f88eb042088 2024-12-07T18:15:38,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/0a09572afd9d473eb81a4d6dfd7ea960 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/0a09572afd9d473eb81a4d6dfd7ea960 2024-12-07T18:15:38,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/f404a08b5fa943cb8f79f874a7dfb2c0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/f404a08b5fa943cb8f79f874a7dfb2c0 2024-12-07T18:15:38,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/24b33633582f4d04b59a65d85aad9c9d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/24b33633582f4d04b59a65d85aad9c9d 2024-12-07T18:15:38,493 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ea8b82784efa4737b97e19cf531021ab to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/ea8b82784efa4737b97e19cf531021ab 2024-12-07T18:15:38,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/51aa6fd313804281bff7a10c347a9e08 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/51aa6fd313804281bff7a10c347a9e08 2024-12-07T18:15:38,494 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/71a7573988944da284371daebea23b8c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/71a7573988944da284371daebea23b8c 2024-12-07T18:15:38,495 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/b858ec743fe146a1b07810d95369ccc1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/b858ec743fe146a1b07810d95369ccc1 2024-12-07T18:15:38,496 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/45ef4496ba6f43cda6cce9aecbae66f0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/45ef4496ba6f43cda6cce9aecbae66f0 2024-12-07T18:15:38,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/04466b563e284746b2d23edbd1b88f5b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/04466b563e284746b2d23edbd1b88f5b 2024-12-07T18:15:38,497 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/4b4230c9e50b49b0bf412fac571e1a59 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/4b4230c9e50b49b0bf412fac571e1a59 2024-12-07T18:15:38,498 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/89a2a6d5aa494de6a12496850578654d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/89a2a6d5aa494de6a12496850578654d 2024-12-07T18:15:38,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/490091c4fe4845da88dfe1cd3bc3c264 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/490091c4fe4845da88dfe1cd3bc3c264 2024-12-07T18:15:38,499 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/5baa7f24d11e4d86929dc6a6403ccc50 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/5baa7f24d11e4d86929dc6a6403ccc50 2024-12-07T18:15:38,500 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/48f1c516ac3245e38b3ae0ee6084f512 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/48f1c516ac3245e38b3ae0ee6084f512 2024-12-07T18:15:38,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/40d66bff080a4f32a441fef8dc85eb25 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/40d66bff080a4f32a441fef8dc85eb25 2024-12-07T18:15:38,501 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/08fc1133d9ad47c2b79b2237aadd61e7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/08fc1133d9ad47c2b79b2237aadd61e7 2024-12-07T18:15:38,502 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8c26e4d074c74fe09a8e21d857eeefb0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/8c26e4d074c74fe09a8e21d857eeefb0 2024-12-07T18:15:38,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/9da5715465af4249ab20fc43f934b929 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/9da5715465af4249ab20fc43f934b929 2024-12-07T18:15:38,503 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/7824080818334d65a5520a09b4124c5d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/7824080818334d65a5520a09b4124c5d 2024-12-07T18:15:38,504 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/191d1395b96049c3ba3fe3057b423f78 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/191d1395b96049c3ba3fe3057b423f78 2024-12-07T18:15:38,505 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6d0829168b77492bbf22f15441b01e4a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/885e9da2a3bd499387a97f7cb8a0f37d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/1b9351f1bce343f3a7b5f2734149236d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/47d65eeab47c40b8bdec62da8baffa43, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/fbf44b8c9472471fa810f4df0a3fb943, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/81ce03d58f3b4c3abf225762084367ff, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3b2b15478a804249bed86b869c9ab0d9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/219b1c1a850843e285075d514dcdd5d5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/59fb1951bd924212b1a949722845e542, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e60a72c17e24580ab83b553dff807fb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/579bb8ef40a74793b0b51fbd87f31743, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/b53761101050478884c911d71d76d848, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3577053d276e41108a435a45d59c7054, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/ef31241a21e14b198de2358e9a7e890b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/402316b596fe4365b68836ebe736c550, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a2d40686099e4de9bb9226b8f8bb08b7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/36479405c3154b45882e25f7c91c5dcb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/786d31e66d8040c6a375c29c93bcc913, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/df26db9bd3344ff79d8cd255917e5ae1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6431421333714e3aa5fe961895a542b6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/7494e03440224e57a2e66ee8538d1d2d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e652088c519459ba37956d3187e3b96, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3f2a2b28327543b58d5f145b5bb0aebc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a29cff4fc9944b6da398f66149323829] to archive 2024-12-07T18:15:38,506 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:15:38,507 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6d0829168b77492bbf22f15441b01e4a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6d0829168b77492bbf22f15441b01e4a 2024-12-07T18:15:38,508 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/885e9da2a3bd499387a97f7cb8a0f37d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/885e9da2a3bd499387a97f7cb8a0f37d 2024-12-07T18:15:38,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/1b9351f1bce343f3a7b5f2734149236d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/1b9351f1bce343f3a7b5f2734149236d 2024-12-07T18:15:38,509 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/47d65eeab47c40b8bdec62da8baffa43 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/47d65eeab47c40b8bdec62da8baffa43 2024-12-07T18:15:38,510 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/fbf44b8c9472471fa810f4df0a3fb943 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/fbf44b8c9472471fa810f4df0a3fb943 2024-12-07T18:15:38,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/81ce03d58f3b4c3abf225762084367ff to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/81ce03d58f3b4c3abf225762084367ff 2024-12-07T18:15:38,511 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3b2b15478a804249bed86b869c9ab0d9 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3b2b15478a804249bed86b869c9ab0d9 2024-12-07T18:15:38,512 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/219b1c1a850843e285075d514dcdd5d5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/219b1c1a850843e285075d514dcdd5d5 2024-12-07T18:15:38,513 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/59fb1951bd924212b1a949722845e542 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/59fb1951bd924212b1a949722845e542 2024-12-07T18:15:38,514 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e60a72c17e24580ab83b553dff807fb to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e60a72c17e24580ab83b553dff807fb 2024-12-07T18:15:38,515 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/579bb8ef40a74793b0b51fbd87f31743 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/579bb8ef40a74793b0b51fbd87f31743 2024-12-07T18:15:38,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/b53761101050478884c911d71d76d848 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/b53761101050478884c911d71d76d848 2024-12-07T18:15:38,516 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3577053d276e41108a435a45d59c7054 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3577053d276e41108a435a45d59c7054 2024-12-07T18:15:38,517 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/ef31241a21e14b198de2358e9a7e890b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/ef31241a21e14b198de2358e9a7e890b 2024-12-07T18:15:38,518 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/402316b596fe4365b68836ebe736c550 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/402316b596fe4365b68836ebe736c550 2024-12-07T18:15:38,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a2d40686099e4de9bb9226b8f8bb08b7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a2d40686099e4de9bb9226b8f8bb08b7 2024-12-07T18:15:38,519 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/36479405c3154b45882e25f7c91c5dcb to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/36479405c3154b45882e25f7c91c5dcb 2024-12-07T18:15:38,520 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/786d31e66d8040c6a375c29c93bcc913 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/786d31e66d8040c6a375c29c93bcc913 2024-12-07T18:15:38,521 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/df26db9bd3344ff79d8cd255917e5ae1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/df26db9bd3344ff79d8cd255917e5ae1 2024-12-07T18:15:38,522 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6431421333714e3aa5fe961895a542b6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6431421333714e3aa5fe961895a542b6 2024-12-07T18:15:38,523 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/7494e03440224e57a2e66ee8538d1d2d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/7494e03440224e57a2e66ee8538d1d2d 2024-12-07T18:15:38,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e652088c519459ba37956d3187e3b96 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/6e652088c519459ba37956d3187e3b96 2024-12-07T18:15:38,524 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3f2a2b28327543b58d5f145b5bb0aebc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/3f2a2b28327543b58d5f145b5bb0aebc 2024-12-07T18:15:38,525 DEBUG [StoreCloser-TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a29cff4fc9944b6da398f66149323829 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/a29cff4fc9944b6da398f66149323829 2024-12-07T18:15:38,529 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/recovered.edits/448.seqid, newMaxSeqId=448, maxSeqId=4 2024-12-07T18:15:38,529 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. 2024-12-07T18:15:38,529 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 6f3992ce41b26fed1c9fca93d87e0364: Waiting for close lock at 1733595337014Running coprocessor pre-close hooks at 1733595337014Disabling compacts and flushes for region at 1733595337014Disabling writes for close at 1733595337162 (+148 ms)Obtaining lock to block concurrent updates at 1733595337162Preparing flush snapshotting stores in 6f3992ce41b26fed1c9fca93d87e0364 at 1733595337162Finished memstore snapshotting TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364., syncing WAL and waiting on mvcc, flushsize=dataSize=26265, getHeapSize=64968, getOffHeapSize=0, getCellsCount=573 at 1733595337162Flushing stores of TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364. at 1733595337163 (+1 ms)Flushing 6f3992ce41b26fed1c9fca93d87e0364/A: creating writer at 1733595337163Flushing mob file 6f3992ce41b26fed1c9fca93d87e0364/A: appending metadata at 1733595337178 (+15 ms)Flushing mob file 6f3992ce41b26fed1c9fca93d87e0364/A: closing flushed file at 1733595337178Flushing 6f3992ce41b26fed1c9fca93d87e0364/A: appending metadata at 1733595337620 (+442 ms)Flushing 6f3992ce41b26fed1c9fca93d87e0364/A: closing flushed file at 1733595337620Flushing 6f3992ce41b26fed1c9fca93d87e0364/B: creating writer at 1733595337629 (+9 ms)Flushing 6f3992ce41b26fed1c9fca93d87e0364/B: appending metadata at 1733595337632 (+3 ms)Flushing 6f3992ce41b26fed1c9fca93d87e0364/B: closing flushed file at 1733595337632Flushing 6f3992ce41b26fed1c9fca93d87e0364/C: creating writer at 1733595338042 (+410 ms)Flushing 6f3992ce41b26fed1c9fca93d87e0364/C: appending metadata at 1733595338045 (+3 ms)Flushing 6f3992ce41b26fed1c9fca93d87e0364/C: closing flushed file at 1733595338045Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53bc4a03: reopening flushed file at 1733595338452 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44783889: reopening flushed file at 1733595338455 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67b6686e: reopening flushed file at 1733595338459 (+4 ms)Finished flush of dataSize ~25.65 KB/26265, heapSize ~63.45 KB/64968, currentSize=0 B/0 for 6f3992ce41b26fed1c9fca93d87e0364 in 1301ms, sequenceid=445, compaction requested=true at 1733595338463 (+4 ms)Writing region close event to WAL at 1733595338526 (+63 ms)Running coprocessor post-close hooks at 1733595338529 (+3 ms)Closed at 1733595338529 2024-12-07T18:15:38,531 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:38,531 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=6f3992ce41b26fed1c9fca93d87e0364, regionState=CLOSED 2024-12-07T18:15:38,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:15:38,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-07T18:15:38,535 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 6f3992ce41b26fed1c9fca93d87e0364, server=92acd7e2389b,33947,1733595217075 in 1.6730 sec 2024-12-07T18:15:38,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-12-07T18:15:38,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6f3992ce41b26fed1c9fca93d87e0364, UNASSIGN in 1.6790 sec 2024-12-07T18:15:38,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-07T18:15:38,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6820 sec 2024-12-07T18:15:38,538 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595338538"}]},"ts":"1733595338538"} 2024-12-07T18:15:38,539 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T18:15:38,539 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T18:15:38,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6930 sec 2024-12-07T18:15:38,925 WARN [regionserver/92acd7e2389b:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-12-07T18:15:39,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-07T18:15:39,009 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:39,009 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T18:15:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:39,011 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-07T18:15:39,011 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:39,013 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,015 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/recovered.edits] 2024-12-07T18:15:39,017 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1a5f430919cb4d51843f3d12d3706bbe to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/1a5f430919cb4d51843f3d12d3706bbe 2024-12-07T18:15:39,018 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2d98659356d04bd180ed9078ab603c91 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/2d98659356d04bd180ed9078ab603c91 2024-12-07T18:15:39,018 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/5f1581fa9250413da3cdf5b24db34541 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/A/5f1581fa9250413da3cdf5b24db34541 2024-12-07T18:15:39,020 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/22369dac6a3a486880d831e881a0feef to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/22369dac6a3a486880d831e881a0feef 2024-12-07T18:15:39,021 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/7a1b07bc736e448585ee9aff2a031f54 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/7a1b07bc736e448585ee9aff2a031f54 2024-12-07T18:15:39,022 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/dcff1148526643c59d4ad818c0ac053e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/B/dcff1148526643c59d4ad818c0ac053e 2024-12-07T18:15:39,023 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/2a966fd5621a4eef80f5cdf0993ffb3e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/2a966fd5621a4eef80f5cdf0993ffb3e 2024-12-07T18:15:39,024 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/da572fd36de145c4950056a224b64ce3 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/da572fd36de145c4950056a224b64ce3 2024-12-07T18:15:39,025 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/f5e324eafa8b4f4fa27bd8591242d8af to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/C/f5e324eafa8b4f4fa27bd8591242d8af 2024-12-07T18:15:39,027 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/recovered.edits/448.seqid to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364/recovered.edits/448.seqid 2024-12-07T18:15:39,027 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,028 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T18:15:39,028 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T18:15:39,029 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-07T18:15:39,031 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120707ee97bd59ba4a0296fe73e3c8a67ec7_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120707ee97bd59ba4a0296fe73e3c8a67ec7_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,032 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207103f6a3c533a4e2c9f43e89daa3c29c9_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207103f6a3c533a4e2c9f43e89daa3c29c9_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,033 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072dad870d0ad44aa38c31220fd8997bee_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412072dad870d0ad44aa38c31220fd8997bee_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,034 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073086602e600f450bb9573b2498ad4c08_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073086602e600f450bb9573b2498ad4c08_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,034 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120739f39b036123419db5781bf034a1484b_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120739f39b036123419db5781bf034a1484b_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,035 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120747dc412aa09a4a1a9951ea3dbf9f1d3d_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120747dc412aa09a4a1a9951ea3dbf9f1d3d_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,036 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076793363a521f406696038c33e4ea8b6a_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412076793363a521f406696038c33e4ea8b6a_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,037 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120769e57043e566430bb67326287ba2e825_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120769e57043e566430bb67326287ba2e825_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,038 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412078260765c80574cecb1855e0796ce8151_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412078260765c80574cecb1855e0796ce8151_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,039 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207849981e3ff684f799b5822091341dc60_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207849981e3ff684f799b5822091341dc60_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,040 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412078f3dbb259d454226b511bb375569099d_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412078f3dbb259d454226b511bb375569099d_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,041 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207946c7da4bbf343678acc37a9fce7e6d0_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207946c7da4bbf343678acc37a9fce7e6d0_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,042 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412079fc10ae70c9d4fcda6266cbf559c0b33_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412079fc10ae70c9d4fcda6266cbf559c0b33_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,043 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a92b7b39388d47ecb3163bd111f7e333_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a92b7b39388d47ecb3163bd111f7e333_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,043 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b7213132d8bc4f17a20d46563ab701b8_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207b7213132d8bc4f17a20d46563ab701b8_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,044 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207bf80eee1760d4e489127a282c3f86d64_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207bf80eee1760d4e489127a282c3f86d64_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,045 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c7087d43c19a4ed4b3b9f7406de57ce6_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c7087d43c19a4ed4b3b9f7406de57ce6_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,046 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e6d2346965b74e8ea6c240de7a764a1d_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e6d2346965b74e8ea6c240de7a764a1d_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,047 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e7ac84eb0af94e0b92bf965aa85b4eb9_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e7ac84eb0af94e0b92bf965aa85b4eb9_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,048 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207f35434651034405dab91ef15b955d26a_6f3992ce41b26fed1c9fca93d87e0364 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207f35434651034405dab91ef15b955d26a_6f3992ce41b26fed1c9fca93d87e0364 2024-12-07T18:15:39,048 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T18:15:39,050 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:39,052 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T18:15:39,053 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T18:15:39,054 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:39,054 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T18:15:39,054 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733595339054"}]},"ts":"9223372036854775807"} 2024-12-07T18:15:39,056 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T18:15:39,056 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6f3992ce41b26fed1c9fca93d87e0364, NAME => 'TestAcidGuarantees,,1733595313187.6f3992ce41b26fed1c9fca93d87e0364.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T18:15:39,056 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T18:15:39,056 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733595339056"}]},"ts":"9223372036854775807"} 2024-12-07T18:15:39,057 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T18:15:39,058 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:39,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-12-07T18:15:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-07T18:15:39,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T18:15:39,119 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:39,128 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 250) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-823063722_22 at /127.0.0.1:51748 [Waiting for operation #1212] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-17801953_22 at /127.0.0.1:42702 [Waiting for operation #1069] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-17801953_22 at /127.0.0.1:42398 [Waiting for operation #769] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-823063722_22 at /127.0.0.1:42382 [Waiting for operation #761] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/92acd7e2389b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=430 (was 490), ProcessCount=11 (was 11), AvailableMemoryMB=5922 (was 6542) 2024-12-07T18:15:39,137 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=430, ProcessCount=11, AvailableMemoryMB=5922 2024-12-07T18:15:39,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T18:15:39,138 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:15:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T18:15:39,140 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T18:15:39,140 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:39,140 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-12-07T18:15:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-07T18:15:39,141 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T18:15:39,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742286_1462 (size=1061) 2024-12-07T18:15:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-07T18:15:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-07T18:15:39,547 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68 2024-12-07T18:15:39,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742287_1463 (size=53) 2024-12-07T18:15:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-07T18:15:39,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:15:39,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0fcb6359e763d0a744acdb75bfd0a652, disabling compactions & flushes 2024-12-07T18:15:39,952 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:39,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:39,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. after waiting 0 ms 2024-12-07T18:15:39,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:39,953 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:39,953 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0fcb6359e763d0a744acdb75bfd0a652: Waiting for close lock at 1733595339952Disabling compacts and flushes for region at 1733595339952Disabling writes for close at 1733595339953 (+1 ms)Writing region close event to WAL at 1733595339953Closed at 1733595339953 2024-12-07T18:15:39,954 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T18:15:39,954 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733595339954"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733595339954"}]},"ts":"1733595339954"} 2024-12-07T18:15:39,955 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T18:15:39,956 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T18:15:39,956 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595339956"}]},"ts":"1733595339956"} 2024-12-07T18:15:39,957 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T18:15:39,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fcb6359e763d0a744acdb75bfd0a652, ASSIGN}] 2024-12-07T18:15:39,958 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fcb6359e763d0a744acdb75bfd0a652, ASSIGN 2024-12-07T18:15:39,958 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fcb6359e763d0a744acdb75bfd0a652, ASSIGN; state=OFFLINE, location=92acd7e2389b,33947,1733595217075; forceNewPlan=false, retain=false 2024-12-07T18:15:40,109 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=0fcb6359e763d0a744acdb75bfd0a652, regionState=OPENING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:40,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fcb6359e763d0a744acdb75bfd0a652, ASSIGN because future has completed 2024-12-07T18:15:40,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:15:40,264 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:40,264 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:15:40,265 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,265 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:15:40,265 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,265 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,266 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,267 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:15:40,268 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fcb6359e763d0a744acdb75bfd0a652 columnFamilyName A 2024-12-07T18:15:40,268 DEBUG [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:40,268 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] regionserver.HStore(327): Store=0fcb6359e763d0a744acdb75bfd0a652/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:15:40,268 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,269 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:15:40,269 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fcb6359e763d0a744acdb75bfd0a652 columnFamilyName B 2024-12-07T18:15:40,269 DEBUG [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:40,269 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] regionserver.HStore(327): Store=0fcb6359e763d0a744acdb75bfd0a652/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:15:40,269 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,270 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:15:40,270 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fcb6359e763d0a744acdb75bfd0a652 columnFamilyName C 2024-12-07T18:15:40,270 DEBUG [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:15:40,270 INFO [StoreOpener-0fcb6359e763d0a744acdb75bfd0a652-1 {}] regionserver.HStore(327): Store=0fcb6359e763d0a744acdb75bfd0a652/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:15:40,271 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:40,271 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,271 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,271 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,272 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,272 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,272 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:15:40,273 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,274 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:15:40,275 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 0fcb6359e763d0a744acdb75bfd0a652; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66918157, jitterRate=-0.002841755747795105}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:15:40,275 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:40,275 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 0fcb6359e763d0a744acdb75bfd0a652: Running coprocessor pre-open hook at 1733595340265Writing region info on filesystem at 1733595340265Initializing all the Stores at 1733595340266 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595340266Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595340266Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595340266Cleaning up temporary data from old regions at 1733595340272 (+6 ms)Running coprocessor post-open hooks at 1733595340275 (+3 ms)Region opened successfully at 1733595340275 2024-12-07T18:15:40,276 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., pid=122, masterSystemTime=1733595340262 2024-12-07T18:15:40,277 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:40,277 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:40,278 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=0fcb6359e763d0a744acdb75bfd0a652, regionState=OPEN, openSeqNum=2, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:40,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-07T18:15:40,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:15:40,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-07T18:15:40,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 in 170 msec 2024-12-07T18:15:40,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-12-07T18:15:40,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fcb6359e763d0a744acdb75bfd0a652, ASSIGN in 324 msec 2024-12-07T18:15:40,283 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T18:15:40,283 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595340283"}]},"ts":"1733595340283"} 2024-12-07T18:15:40,284 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T18:15:40,284 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T18:15:40,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-12-07T18:15:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-12-07T18:15:41,288 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:41,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8abc8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,290 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,291 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a698d89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,292 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c55f51e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,293 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,294 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,295 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T18:15:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 92acd7e2389b,38531,1733595216342 2024-12-07T18:15:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4188df36 2024-12-07T18:15:41,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T18:15:41,296 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T18:15:41,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,298 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,298 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7076a62a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,299 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,300 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b73271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,301 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a858a77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,302 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a248b72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,303 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa951f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,304 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700a756f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,306 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53720df8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c095ed7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,308 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a27151c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,309 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18feb9ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,311 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2309ee9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,311 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,312 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f88ea85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,313 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b7f703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,314 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@599d7eac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,316 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56693319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,316 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,317 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af83f79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,319 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25736301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,320 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2af348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,321 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,321 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:15:41,321 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:15:41,321 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:15:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:15:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:15:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b58183d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:15:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:15:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:15:41,322 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:15:41,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23e59a6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:15:41,323 DEBUG [Thread-2072 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,323 DEBUG [Thread-2087 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,323 DEBUG [Thread-2081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,323 DEBUG [Thread-2076 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,323 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,324 DEBUG [Thread-2070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,324 DEBUG [Thread-2083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,324 DEBUG [Thread-2074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-07T18:15:41,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,326 DEBUG [Thread-2089 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,326 DEBUG [Thread-2078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,326 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:41,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-07T18:15:41,326 DEBUG [Thread-2085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:15:41,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,327 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:15:41,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:15:41,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:41,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:15:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:15:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:15:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:15:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:15:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:15:41,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:15:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:15:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:15:41,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:15:41,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2] 2024-12-07T18:15:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:41,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-07T18:15:41,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:41,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:41,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:41,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:41,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:41,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:41,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/fd4a3465611846498bce02b85268fb91 is 50, key is test_row_0/A:col10/1733595341333/Put/seqid=0 2024-12-07T18:15:41,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742288_1464 (size=16681) 2024-12-07T18:15:41,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733595341360, max timestamp=1733595341380Num uniques 859; 2024-12-07T18:15:41,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-07T18:15:41,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733595341360, max timestamp=1733595341380Num uniques 859; 2024-12-07T18:15:41,383 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-07T18:15:41,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733595341360, max timestamp=1733595341380Num uniques 859; 2024-12-07T18:15:41,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-07T18:15:41,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595401407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595401409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595401410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595401411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595401412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-07T18:15:41,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-07T18:15:41,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:41,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:41,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:41,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595401528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595401528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595401528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595401528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595401528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-07T18:15:41,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:41,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:41,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:41,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-07T18:15:41,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595401748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595401748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595401749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595401749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595401749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:41,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/fd4a3465611846498bce02b85268fb91 2024-12-07T18:15:41,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-07T18:15:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:41,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/c552217def7f4d59865798ed863aa2ae is 50, key is test_row_0/B:col10/1733595341333/Put/seqid=0 2024-12-07T18:15:41,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742289_1465 (size=12001) 2024-12-07T18:15:41,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-07T18:15:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:41,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-07T18:15:42,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595402068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:42,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595402068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:42,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595402069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595402069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:42,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595402069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:42,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-07T18:15:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:42,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:42,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:42,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/c552217def7f4d59865798ed863aa2ae 2024-12-07T18:15:42,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/3a7d80046bfa48f9bf62b8982fae8c24 is 50, key is test_row_0/C:col10/1733595341333/Put/seqid=0 2024-12-07T18:15:42,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742290_1466 (size=12001) 2024-12-07T18:15:42,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-07T18:15:42,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:42,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:42,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:42,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-07T18:15:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:42,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-07T18:15:42,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-07T18:15:42,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:42,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:42,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:42,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:42,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:42,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595402579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595402579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:42,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595402588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:42,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:42,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595402588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:42,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595402588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:42,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:42,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/3a7d80046bfa48f9bf62b8982fae8c24 2024-12-07T18:15:42,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/fd4a3465611846498bce02b85268fb91 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/fd4a3465611846498bce02b85268fb91 2024-12-07T18:15:42,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/fd4a3465611846498bce02b85268fb91, entries=250, sequenceid=12, filesize=16.3 K 2024-12-07T18:15:42,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/c552217def7f4d59865798ed863aa2ae as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c552217def7f4d59865798ed863aa2ae 2024-12-07T18:15:42,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c552217def7f4d59865798ed863aa2ae, entries=150, sequenceid=12, filesize=11.7 K 2024-12-07T18:15:42,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/3a7d80046bfa48f9bf62b8982fae8c24 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/3a7d80046bfa48f9bf62b8982fae8c24 2024-12-07T18:15:42,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/3a7d80046bfa48f9bf62b8982fae8c24, entries=150, sequenceid=12, filesize=11.7 K 2024-12-07T18:15:42,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 0fcb6359e763d0a744acdb75bfd0a652 in 1309ms, sequenceid=12, compaction requested=false 2024-12-07T18:15:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-07T18:15:42,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:42,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-07T18:15:42,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:42,706 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-12-07T18:15:42,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:42,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:42,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:42,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:42,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:42,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:15:42,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/145fa59fd6a5404ba3abe2cf5f222f3d is 50, key is test_row_0/A:col10/1733595341409/Put/seqid=0 2024-12-07T18:15:42,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742291_1467 (size=12001) 2024-12-07T18:15:43,118 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/145fa59fd6a5404ba3abe2cf5f222f3d 2024-12-07T18:15:43,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/bd8d7ea66bbb48169f60e7a637fbbbe5 is 50, key is test_row_0/B:col10/1733595341409/Put/seqid=0 2024-12-07T18:15:43,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742292_1468 (size=12001) 2024-12-07T18:15:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-07T18:15:43,531 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/bd8d7ea66bbb48169f60e7a637fbbbe5 2024-12-07T18:15:43,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/ff3921a3f3a048e79fb83564754d65bd is 50, key is test_row_0/C:col10/1733595341409/Put/seqid=0 2024-12-07T18:15:43,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742293_1469 (size=12001) 2024-12-07T18:15:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:43,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:43,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595403618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595403620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595403621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595403622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595403624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595403728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595403749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595403749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595403749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595403749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595403938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:43,946 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/ff3921a3f3a048e79fb83564754d65bd 2024-12-07T18:15:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:43,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/145fa59fd6a5404ba3abe2cf5f222f3d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/145fa59fd6a5404ba3abe2cf5f222f3d 2024-12-07T18:15:43,953 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/145fa59fd6a5404ba3abe2cf5f222f3d, entries=150, sequenceid=45, filesize=11.7 K 2024-12-07T18:15:43,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/bd8d7ea66bbb48169f60e7a637fbbbe5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/bd8d7ea66bbb48169f60e7a637fbbbe5 2024-12-07T18:15:43,957 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/bd8d7ea66bbb48169f60e7a637fbbbe5, entries=150, sequenceid=45, filesize=11.7 K 2024-12-07T18:15:43,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/ff3921a3f3a048e79fb83564754d65bd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ff3921a3f3a048e79fb83564754d65bd 2024-12-07T18:15:43,961 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ff3921a3f3a048e79fb83564754d65bd, entries=150, sequenceid=45, filesize=11.7 K 2024-12-07T18:15:43,962 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 0fcb6359e763d0a744acdb75bfd0a652 in 1256ms, sequenceid=45, compaction requested=false 2024-12-07T18:15:43,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:43,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:43,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-07T18:15:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-07T18:15:43,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-07T18:15:43,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6360 sec 2024-12-07T18:15:43,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.6420 sec 2024-12-07T18:15:43,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:43,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T18:15:43,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:43,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:43,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:43,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:43,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:43,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:43,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/1af6078cca164d049c50d6116a0eac94 is 50, key is test_row_0/A:col10/1733595343622/Put/seqid=0 2024-12-07T18:15:43,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742294_1470 (size=16681) 2024-12-07T18:15:44,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595404008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595404009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595404017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595404019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,048 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T18:15:44,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595404128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595404128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595404128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,133 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,134 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595404138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595404258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595404338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595404338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595404338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595404359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/1af6078cca164d049c50d6116a0eac94 2024-12-07T18:15:44,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/eeb3bcf4b60e4a6bac6a81e26ec8e321 is 50, key is test_row_0/B:col10/1733595343622/Put/seqid=0 2024-12-07T18:15:44,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742295_1471 (size=12001) 2024-12-07T18:15:44,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595404659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595404659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595404659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595404678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595404778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:44,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/eeb3bcf4b60e4a6bac6a81e26ec8e321 2024-12-07T18:15:44,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/ebff422f90bc440ea465f84de8877115 is 50, key is test_row_0/C:col10/1733595343622/Put/seqid=0 2024-12-07T18:15:44,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742296_1472 (size=12001) 2024-12-07T18:15:45,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595405178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:45,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:45,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595405178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:45,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:45,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:45,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595405178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:45,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/ebff422f90bc440ea465f84de8877115 2024-12-07T18:15:45,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/1af6078cca164d049c50d6116a0eac94 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/1af6078cca164d049c50d6116a0eac94 2024-12-07T18:15:45,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595405198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:45,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:45,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/1af6078cca164d049c50d6116a0eac94, entries=250, sequenceid=58, filesize=16.3 K 2024-12-07T18:15:45,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/eeb3bcf4b60e4a6bac6a81e26ec8e321 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/eeb3bcf4b60e4a6bac6a81e26ec8e321 2024-12-07T18:15:45,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/eeb3bcf4b60e4a6bac6a81e26ec8e321, entries=150, sequenceid=58, filesize=11.7 K 2024-12-07T18:15:45,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/ebff422f90bc440ea465f84de8877115 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebff422f90bc440ea465f84de8877115 2024-12-07T18:15:45,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebff422f90bc440ea465f84de8877115, entries=150, sequenceid=58, filesize=11.7 K 2024-12-07T18:15:45,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 0fcb6359e763d0a744acdb75bfd0a652 in 1245ms, sequenceid=58, compaction requested=true 2024-12-07T18:15:45,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:45,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:45,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:45,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:45,220 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:45,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:45,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:45,220 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:45,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:45,220 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:45,221 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/A is initiating minor compaction (all files) 2024-12-07T18:15:45,221 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/A in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:45,221 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:45,221 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/fd4a3465611846498bce02b85268fb91, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/145fa59fd6a5404ba3abe2cf5f222f3d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/1af6078cca164d049c50d6116a0eac94] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=44.3 K 2024-12-07T18:15:45,221 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/B is initiating minor compaction (all files) 2024-12-07T18:15:45,221 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/B in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:45,221 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c552217def7f4d59865798ed863aa2ae, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/bd8d7ea66bbb48169f60e7a637fbbbe5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/eeb3bcf4b60e4a6bac6a81e26ec8e321] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=35.2 K 2024-12-07T18:15:45,221 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd4a3465611846498bce02b85268fb91, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733595341332 2024-12-07T18:15:45,221 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting c552217def7f4d59865798ed863aa2ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733595341333 2024-12-07T18:15:45,221 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145fa59fd6a5404ba3abe2cf5f222f3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733595341409 2024-12-07T18:15:45,222 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8d7ea66bbb48169f60e7a637fbbbe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733595341409 2024-12-07T18:15:45,222 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1af6078cca164d049c50d6116a0eac94, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733595343606 2024-12-07T18:15:45,222 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting eeb3bcf4b60e4a6bac6a81e26ec8e321, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733595343606 2024-12-07T18:15:45,229 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:45,230 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/855d1cc268c7476f8f5d8682dbdc1b59 is 50, key is test_row_0/B:col10/1733595343622/Put/seqid=0 2024-12-07T18:15:45,232 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#A#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:45,232 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/90988ac2a7ec449a934e214cf244d357 is 50, key is test_row_0/A:col10/1733595343622/Put/seqid=0 2024-12-07T18:15:45,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742297_1473 (size=12104) 2024-12-07T18:15:45,239 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/855d1cc268c7476f8f5d8682dbdc1b59 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/855d1cc268c7476f8f5d8682dbdc1b59 2024-12-07T18:15:45,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742298_1474 (size=12104) 2024-12-07T18:15:45,244 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/B of 0fcb6359e763d0a744acdb75bfd0a652 into 855d1cc268c7476f8f5d8682dbdc1b59(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:45,244 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:45,244 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/B, priority=13, startTime=1733595345220; duration=0sec 2024-12-07T18:15:45,244 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:45,244 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:B 2024-12-07T18:15:45,244 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:45,248 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/90988ac2a7ec449a934e214cf244d357 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/90988ac2a7ec449a934e214cf244d357 2024-12-07T18:15:45,248 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:45,248 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/C is initiating minor compaction (all files) 2024-12-07T18:15:45,248 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/C in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:45,249 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/3a7d80046bfa48f9bf62b8982fae8c24, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ff3921a3f3a048e79fb83564754d65bd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebff422f90bc440ea465f84de8877115] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=35.2 K 2024-12-07T18:15:45,249 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a7d80046bfa48f9bf62b8982fae8c24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733595341333 2024-12-07T18:15:45,249 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ff3921a3f3a048e79fb83564754d65bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1733595341409 2024-12-07T18:15:45,250 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ebff422f90bc440ea465f84de8877115, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733595343606 2024-12-07T18:15:45,253 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/A of 0fcb6359e763d0a744acdb75bfd0a652 into 90988ac2a7ec449a934e214cf244d357(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:45,253 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:45,253 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/A, priority=13, startTime=1733595345220; duration=0sec 2024-12-07T18:15:45,253 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:45,253 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:A 2024-12-07T18:15:45,256 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:45,257 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/901c7e114572491c9b3206525d6e1006 is 50, key is test_row_0/C:col10/1733595343622/Put/seqid=0 2024-12-07T18:15:45,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742299_1475 (size=12104) 2024-12-07T18:15:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-07T18:15:45,488 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:45,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-07T18:15:45,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-07T18:15:45,490 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:45,491 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:45,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-07T18:15:45,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-07T18:15:45,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:45,643 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T18:15:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:45,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/c3bd0120a41142c7a7b27569b30372e5 is 50, key is test_row_0/A:col10/1733595344017/Put/seqid=0 2024-12-07T18:15:45,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742300_1476 (size=12001) 2024-12-07T18:15:45,674 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/901c7e114572491c9b3206525d6e1006 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/901c7e114572491c9b3206525d6e1006 2024-12-07T18:15:45,678 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/C of 0fcb6359e763d0a744acdb75bfd0a652 into 901c7e114572491c9b3206525d6e1006(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:45,678 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:45,678 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/C, priority=13, startTime=1733595345220; duration=0sec 2024-12-07T18:15:45,678 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:45,678 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:C 2024-12-07T18:15:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:45,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-07T18:15:45,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595405869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:45,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:45,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595405979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:46,053 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/c3bd0120a41142c7a7b27569b30372e5 2024-12-07T18:15:46,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/3a3eb905e69740aaa078f80772c12fa0 is 50, key is test_row_0/B:col10/1733595344017/Put/seqid=0 2024-12-07T18:15:46,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742301_1477 (size=12001) 2024-12-07T18:15:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-07T18:15:46,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595406188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:46,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595406199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:46,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595406199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:46,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595406199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:46,207 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:46,208 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,208 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:46,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595406218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:46,224 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:46,464 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/3a3eb905e69740aaa078f80772c12fa0 2024-12-07T18:15:46,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/0c9060837caf4d4983aee6efabfd43a7 is 50, key is test_row_0/C:col10/1733595344017/Put/seqid=0 2024-12-07T18:15:46,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742302_1478 (size=12001) 2024-12-07T18:15:46,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595406508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:46,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-07T18:15:46,875 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/0c9060837caf4d4983aee6efabfd43a7 2024-12-07T18:15:46,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/c3bd0120a41142c7a7b27569b30372e5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c3bd0120a41142c7a7b27569b30372e5 2024-12-07T18:15:46,883 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c3bd0120a41142c7a7b27569b30372e5, entries=150, sequenceid=79, filesize=11.7 K 2024-12-07T18:15:46,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/3a3eb905e69740aaa078f80772c12fa0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/3a3eb905e69740aaa078f80772c12fa0 2024-12-07T18:15:46,887 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/3a3eb905e69740aaa078f80772c12fa0, entries=150, sequenceid=79, filesize=11.7 K 2024-12-07T18:15:46,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/0c9060837caf4d4983aee6efabfd43a7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0c9060837caf4d4983aee6efabfd43a7 2024-12-07T18:15:46,891 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0c9060837caf4d4983aee6efabfd43a7, entries=150, sequenceid=79, filesize=11.7 K 2024-12-07T18:15:46,891 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0fcb6359e763d0a744acdb75bfd0a652 in 1248ms, sequenceid=79, compaction requested=false 2024-12-07T18:15:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:46,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-07T18:15:46,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-07T18:15:46,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-07T18:15:46,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4020 sec 2024-12-07T18:15:46,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4060 sec 2024-12-07T18:15:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:47,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T18:15:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:47,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:47,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/23876481f9404559a4b0734c75a3a7bd is 50, key is test_row_0/A:col10/1733595345867/Put/seqid=0 2024-12-07T18:15:47,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742303_1479 (size=12001) 2024-12-07T18:15:47,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/23876481f9404559a4b0734c75a3a7bd 2024-12-07T18:15:47,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/232dcc1e14d14026a2c53c8310dd5f78 is 50, key is test_row_0/B:col10/1733595345867/Put/seqid=0 2024-12-07T18:15:47,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742304_1480 (size=12001) 2024-12-07T18:15:47,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:47,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595407154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:47,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:47,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595407268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:47,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/232dcc1e14d14026a2c53c8310dd5f78 2024-12-07T18:15:47,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/a08c50f2568847c09e52e73f7abd836e is 50, key is test_row_0/C:col10/1733595345867/Put/seqid=0 2024-12-07T18:15:47,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742305_1481 (size=12001) 2024-12-07T18:15:47,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595407478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:47,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-07T18:15:47,638 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:47,638 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-07T18:15:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T18:15:47,640 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:47,640 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:47,641 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T18:15:47,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-07T18:15:47,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:47,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595407789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:47,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:47,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:47,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:47,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:47,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:47,856 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/a08c50f2568847c09e52e73f7abd836e 2024-12-07T18:15:47,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/23876481f9404559a4b0734c75a3a7bd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/23876481f9404559a4b0734c75a3a7bd 2024-12-07T18:15:47,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/23876481f9404559a4b0734c75a3a7bd, entries=150, sequenceid=94, filesize=11.7 K 2024-12-07T18:15:47,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/232dcc1e14d14026a2c53c8310dd5f78 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/232dcc1e14d14026a2c53c8310dd5f78 2024-12-07T18:15:47,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/232dcc1e14d14026a2c53c8310dd5f78, entries=150, sequenceid=94, filesize=11.7 K 2024-12-07T18:15:47,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/a08c50f2568847c09e52e73f7abd836e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a08c50f2568847c09e52e73f7abd836e 2024-12-07T18:15:47,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a08c50f2568847c09e52e73f7abd836e, entries=150, sequenceid=94, filesize=11.7 K 2024-12-07T18:15:47,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0fcb6359e763d0a744acdb75bfd0a652 in 861ms, sequenceid=94, compaction requested=true 2024-12-07T18:15:47,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:47,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:47,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:47,886 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:47,886 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:47,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:47,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:47,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:47,887 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:47,887 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/A is initiating minor compaction (all files) 2024-12-07T18:15:47,887 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/A in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:47,888 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/90988ac2a7ec449a934e214cf244d357, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c3bd0120a41142c7a7b27569b30372e5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/23876481f9404559a4b0734c75a3a7bd] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=35.3 K 2024-12-07T18:15:47,888 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:47,888 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/B is initiating minor compaction (all files) 2024-12-07T18:15:47,888 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/B in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:47,888 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/855d1cc268c7476f8f5d8682dbdc1b59, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/3a3eb905e69740aaa078f80772c12fa0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/232dcc1e14d14026a2c53c8310dd5f78] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=35.3 K 2024-12-07T18:15:47,888 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90988ac2a7ec449a934e214cf244d357, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733595343606 2024-12-07T18:15:47,888 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 855d1cc268c7476f8f5d8682dbdc1b59, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733595343606 2024-12-07T18:15:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:47,889 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3bd0120a41142c7a7b27569b30372e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733595343994 2024-12-07T18:15:47,889 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a3eb905e69740aaa078f80772c12fa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733595343994 2024-12-07T18:15:47,889 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23876481f9404559a4b0734c75a3a7bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733595345810 2024-12-07T18:15:47,889 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 232dcc1e14d14026a2c53c8310dd5f78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733595345810 2024-12-07T18:15:47,904 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:47,904 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#A#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:47,904 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/cbb24907e0d14847be568483648b7f49 is 50, key is test_row_0/B:col10/1733595345867/Put/seqid=0 2024-12-07T18:15:47,904 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/01be43a3ed73470f855a4c633ff9a4b5 is 50, key is test_row_0/A:col10/1733595345867/Put/seqid=0 2024-12-07T18:15:47,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-07T18:15:47,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:47,948 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T18:15:47,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:47,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:47,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:47,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T18:15:47,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742307_1483 (size=12207) 2024-12-07T18:15:47,986 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/01be43a3ed73470f855a4c633ff9a4b5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/01be43a3ed73470f855a4c633ff9a4b5 2024-12-07T18:15:47,991 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/A of 0fcb6359e763d0a744acdb75bfd0a652 into 01be43a3ed73470f855a4c633ff9a4b5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:47,991 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:47,991 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/A, priority=13, startTime=1733595347886; duration=0sec 2024-12-07T18:15:47,991 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:47,991 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:A 2024-12-07T18:15:47,992 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:47,993 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:47,993 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/C is initiating minor compaction (all files) 2024-12-07T18:15:47,993 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/C in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:47,993 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/901c7e114572491c9b3206525d6e1006, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0c9060837caf4d4983aee6efabfd43a7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a08c50f2568847c09e52e73f7abd836e] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=35.3 K 2024-12-07T18:15:47,993 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 901c7e114572491c9b3206525d6e1006, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733595343606 2024-12-07T18:15:47,994 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c9060837caf4d4983aee6efabfd43a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733595343994 2024-12-07T18:15:47,994 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08c50f2568847c09e52e73f7abd836e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733595345810 2024-12-07T18:15:47,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/e1bed2d45b594546bdfb2d2b9f775d5a is 50, key is test_row_0/A:col10/1733595347153/Put/seqid=0 2024-12-07T18:15:48,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742306_1482 (size=12207) 2024-12-07T18:15:48,022 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#C#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:48,023 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/2bdd8d37287b44f380978889fedc907b is 50, key is test_row_0/C:col10/1733595345867/Put/seqid=0 2024-12-07T18:15:48,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742308_1484 (size=12001) 2024-12-07T18:15:48,033 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/e1bed2d45b594546bdfb2d2b9f775d5a 2024-12-07T18:15:48,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/1b06645affdf48cc8ee7a083e3495762 is 50, key is test_row_0/B:col10/1733595347153/Put/seqid=0 2024-12-07T18:15:48,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742309_1485 (size=12207) 2024-12-07T18:15:48,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742310_1486 (size=12001) 2024-12-07T18:15:48,101 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/1b06645affdf48cc8ee7a083e3495762 2024-12-07T18:15:48,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/9c79d4f5dee74e23b38fe971e3b1861f is 50, key is test_row_0/C:col10/1733595347153/Put/seqid=0 2024-12-07T18:15:48,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742311_1487 (size=12001) 2024-12-07T18:15:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:48,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T18:15:48,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595408261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595408265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595408284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595408286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595408298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595408388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595408388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,413 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/cbb24907e0d14847be568483648b7f49 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/cbb24907e0d14847be568483648b7f49 2024-12-07T18:15:48,419 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/B of 0fcb6359e763d0a744acdb75bfd0a652 into cbb24907e0d14847be568483648b7f49(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:48,419 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:48,419 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/B, priority=13, startTime=1733595347886; duration=0sec 2024-12-07T18:15:48,419 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:48,419 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:B 2024-12-07T18:15:48,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595408409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595408409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,486 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/2bdd8d37287b44f380978889fedc907b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/2bdd8d37287b44f380978889fedc907b 2024-12-07T18:15:48,490 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/C of 0fcb6359e763d0a744acdb75bfd0a652 into 2bdd8d37287b44f380978889fedc907b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:48,490 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:48,490 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/C, priority=13, startTime=1733595347887; duration=0sec 2024-12-07T18:15:48,490 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:48,490 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:C 2024-12-07T18:15:48,581 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/9c79d4f5dee74e23b38fe971e3b1861f 2024-12-07T18:15:48,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/e1bed2d45b594546bdfb2d2b9f775d5a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/e1bed2d45b594546bdfb2d2b9f775d5a 2024-12-07T18:15:48,588 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/e1bed2d45b594546bdfb2d2b9f775d5a, entries=150, sequenceid=112, filesize=11.7 K 2024-12-07T18:15:48,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/1b06645affdf48cc8ee7a083e3495762 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/1b06645affdf48cc8ee7a083e3495762 2024-12-07T18:15:48,592 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/1b06645affdf48cc8ee7a083e3495762, entries=150, sequenceid=112, filesize=11.7 K 2024-12-07T18:15:48,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/9c79d4f5dee74e23b38fe971e3b1861f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/9c79d4f5dee74e23b38fe971e3b1861f 2024-12-07T18:15:48,598 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/9c79d4f5dee74e23b38fe971e3b1861f, entries=150, sequenceid=112, filesize=11.7 K 2024-12-07T18:15:48,598 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0fcb6359e763d0a744acdb75bfd0a652 in 650ms, sequenceid=112, compaction requested=false 2024-12-07T18:15:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-07T18:15:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-07T18:15:48,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-07T18:15:48,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 959 msec 2024-12-07T18:15:48,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 964 msec 2024-12-07T18:15:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:48,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T18:15:48,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:48,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:48,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:48,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:48,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:48,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:48,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/71d1ba89e9884738bf515a62a6e619e1 is 50, key is test_row_0/A:col10/1733595348279/Put/seqid=0 2024-12-07T18:15:48,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742312_1488 (size=12101) 2024-12-07T18:15:48,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595408645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595408654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595408655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595408656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595408758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595408768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-07T18:15:48,779 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:48,780 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-07T18:15:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T18:15:48,781 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:48,782 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:48,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595408779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595408779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,790 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T18:15:48,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T18:15:48,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:48,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:48,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:48,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:48,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:48,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595408969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:48,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595408978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595408998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595408998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/71d1ba89e9884738bf515a62a6e619e1 2024-12-07T18:15:49,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/b7c0d87bc67a438e999da08fee80a193 is 50, key is test_row_0/B:col10/1733595348279/Put/seqid=0 2024-12-07T18:15:49,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742313_1489 (size=12101) 2024-12-07T18:15:49,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/b7c0d87bc67a438e999da08fee80a193 2024-12-07T18:15:49,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/17935b92e8194d93b7959b6c21d04fe3 is 50, key is test_row_0/C:col10/1733595348279/Put/seqid=0 2024-12-07T18:15:49,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742314_1490 (size=12101) 2024-12-07T18:15:49,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T18:15:49,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:49,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:49,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:49,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:49,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T18:15:49,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T18:15:49,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:49,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:49,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:49,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:49,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:49,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595409278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595409299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595409318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595409318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595409328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,338 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T18:15:49,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:49,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:49,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:49,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:49,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:49,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T18:15:49,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/17935b92e8194d93b7959b6c21d04fe3 2024-12-07T18:15:49,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/71d1ba89e9884738bf515a62a6e619e1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/71d1ba89e9884738bf515a62a6e619e1 2024-12-07T18:15:49,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/71d1ba89e9884738bf515a62a6e619e1, entries=150, sequenceid=131, filesize=11.8 K 2024-12-07T18:15:49,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/b7c0d87bc67a438e999da08fee80a193 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/b7c0d87bc67a438e999da08fee80a193 2024-12-07T18:15:49,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/b7c0d87bc67a438e999da08fee80a193, entries=150, sequenceid=131, filesize=11.8 K 2024-12-07T18:15:49,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/17935b92e8194d93b7959b6c21d04fe3 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/17935b92e8194d93b7959b6c21d04fe3 2024-12-07T18:15:49,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/17935b92e8194d93b7959b6c21d04fe3, entries=150, sequenceid=131, filesize=11.8 K 2024-12-07T18:15:49,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0fcb6359e763d0a744acdb75bfd0a652 in 849ms, sequenceid=131, compaction requested=true 2024-12-07T18:15:49,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:49,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:49,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:49,466 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:49,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:49,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:49,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:49,466 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:49,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:49,466 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:49,466 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:49,466 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/A is initiating minor compaction (all files) 2024-12-07T18:15:49,466 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/B is initiating minor compaction (all files) 2024-12-07T18:15:49,467 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/A in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:49,467 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/B in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:49,467 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/01be43a3ed73470f855a4c633ff9a4b5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/e1bed2d45b594546bdfb2d2b9f775d5a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/71d1ba89e9884738bf515a62a6e619e1] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=35.5 K 2024-12-07T18:15:49,467 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/cbb24907e0d14847be568483648b7f49, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/1b06645affdf48cc8ee7a083e3495762, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/b7c0d87bc67a438e999da08fee80a193] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=35.5 K 2024-12-07T18:15:49,467 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01be43a3ed73470f855a4c633ff9a4b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733595345810 2024-12-07T18:15:49,467 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting cbb24907e0d14847be568483648b7f49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733595345810 2024-12-07T18:15:49,467 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1bed2d45b594546bdfb2d2b9f775d5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733595347128 2024-12-07T18:15:49,467 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b06645affdf48cc8ee7a083e3495762, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733595347128 2024-12-07T18:15:49,467 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71d1ba89e9884738bf515a62a6e619e1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595348279 2024-12-07T18:15:49,467 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting b7c0d87bc67a438e999da08fee80a193, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595348279 2024-12-07T18:15:49,474 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:49,474 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/c645ed55f6b84d029688cbfd145f0f5e is 50, key is test_row_0/B:col10/1733595348279/Put/seqid=0 2024-12-07T18:15:49,474 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#A#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:49,475 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/b5346ec1dcd64476997d18e1c2a5f50c is 50, key is test_row_0/A:col10/1733595348279/Put/seqid=0 2024-12-07T18:15:49,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742315_1491 (size=12409) 2024-12-07T18:15:49,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742316_1492 (size=12409) 2024-12-07T18:15:49,487 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/b5346ec1dcd64476997d18e1c2a5f50c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/b5346ec1dcd64476997d18e1c2a5f50c 2024-12-07T18:15:49,487 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/c645ed55f6b84d029688cbfd145f0f5e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c645ed55f6b84d029688cbfd145f0f5e 2024-12-07T18:15:49,492 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/B of 0fcb6359e763d0a744acdb75bfd0a652 into c645ed55f6b84d029688cbfd145f0f5e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:49,492 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:49,492 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/B, priority=13, startTime=1733595349466; duration=0sec 2024-12-07T18:15:49,492 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/A of 0fcb6359e763d0a744acdb75bfd0a652 into b5346ec1dcd64476997d18e1c2a5f50c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:49,492 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:49,492 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:B 2024-12-07T18:15:49,492 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:49,492 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/A, priority=13, startTime=1733595349465; duration=0sec 2024-12-07T18:15:49,492 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:49,492 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:49,492 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:A 2024-12-07T18:15:49,493 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:49,493 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/C is initiating minor compaction (all files) 2024-12-07T18:15:49,493 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/C in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:49,493 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/2bdd8d37287b44f380978889fedc907b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/9c79d4f5dee74e23b38fe971e3b1861f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/17935b92e8194d93b7959b6c21d04fe3] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=35.5 K 2024-12-07T18:15:49,493 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bdd8d37287b44f380978889fedc907b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733595345810 2024-12-07T18:15:49,494 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c79d4f5dee74e23b38fe971e3b1861f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1733595347128 2024-12-07T18:15:49,494 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 17935b92e8194d93b7959b6c21d04fe3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595348279 2024-12-07T18:15:49,500 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#C#compaction#413 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:49,500 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/90672bd18309430da81ad5f7be066cef is 50, key is test_row_0/C:col10/1733595348279/Put/seqid=0 2024-12-07T18:15:49,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742317_1493 (size=12409) 2024-12-07T18:15:49,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-07T18:15:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:49,547 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T18:15:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:49,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/0bc217077c474963914f8d243268beca is 50, key is test_row_0/A:col10/1733595348641/Put/seqid=0 2024-12-07T18:15:49,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742318_1494 (size=12151) 2024-12-07T18:15:49,557 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/0bc217077c474963914f8d243268beca 2024-12-07T18:15:49,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/991297f248ac48cfb4da92c558ef8dfe is 50, key is test_row_0/B:col10/1733595348641/Put/seqid=0 2024-12-07T18:15:49,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742319_1495 (size=12151) 2024-12-07T18:15:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:49,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:49,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595409839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595409845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595409848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595409846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,909 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/90672bd18309430da81ad5f7be066cef as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/90672bd18309430da81ad5f7be066cef 2024-12-07T18:15:49,913 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/C of 0fcb6359e763d0a744acdb75bfd0a652 into 90672bd18309430da81ad5f7be066cef(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:49,913 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:49,913 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/C, priority=13, startTime=1733595349466; duration=0sec 2024-12-07T18:15:49,913 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:49,913 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:C 2024-12-07T18:15:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T18:15:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595409958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,968 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/991297f248ac48cfb4da92c558ef8dfe 2024-12-07T18:15:49,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/4d2dd16cf1da42f7819cb8c5db994cc7 is 50, key is test_row_0/C:col10/1733595348641/Put/seqid=0 2024-12-07T18:15:49,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595409968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595409968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595409968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:49,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:49,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742320_1496 (size=12151) 2024-12-07T18:15:50,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595410168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595410188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595410188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595410188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,385 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/4d2dd16cf1da42f7819cb8c5db994cc7 2024-12-07T18:15:50,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/0bc217077c474963914f8d243268beca as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0bc217077c474963914f8d243268beca 2024-12-07T18:15:50,392 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0bc217077c474963914f8d243268beca, entries=150, sequenceid=149, filesize=11.9 K 2024-12-07T18:15:50,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/991297f248ac48cfb4da92c558ef8dfe as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/991297f248ac48cfb4da92c558ef8dfe 2024-12-07T18:15:50,398 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/991297f248ac48cfb4da92c558ef8dfe, entries=150, sequenceid=149, filesize=11.9 K 2024-12-07T18:15:50,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/4d2dd16cf1da42f7819cb8c5db994cc7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/4d2dd16cf1da42f7819cb8c5db994cc7 2024-12-07T18:15:50,402 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/4d2dd16cf1da42f7819cb8c5db994cc7, entries=150, sequenceid=149, filesize=11.9 K 2024-12-07T18:15:50,403 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0fcb6359e763d0a744acdb75bfd0a652 in 856ms, sequenceid=149, compaction requested=false 2024-12-07T18:15:50,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:50,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:50,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-07T18:15:50,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-07T18:15:50,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-07T18:15:50,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6220 sec 2024-12-07T18:15:50,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.6260 sec 2024-12-07T18:15:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:50,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-07T18:15:50,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:50,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:50,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:50,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:50,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:50,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:50,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/0db967f316be4e42acf3ca2a69865f43 is 50, key is test_row_0/A:col10/1733595349844/Put/seqid=0 2024-12-07T18:15:50,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742321_1497 (size=14541) 2024-12-07T18:15:50,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595410527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595410535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595410537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595410538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595410648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595410658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595410659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595410659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595410868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595410878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595410878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595410879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:50,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:50,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/0db967f316be4e42acf3ca2a69865f43 2024-12-07T18:15:50,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/217e8202073f4fe2b572cd43c5b81967 is 50, key is test_row_0/B:col10/1733595349844/Put/seqid=0 2024-12-07T18:15:50,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742322_1498 (size=12151) 2024-12-07T18:15:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-07T18:15:50,928 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:50,929 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-07T18:15:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T18:15:50,931 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:50,931 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:50,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T18:15:51,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-07T18:15:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595411188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:51,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595411198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:51,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595411199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595411199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:51,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-07T18:15:51,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:51,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T18:15:51,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/217e8202073f4fe2b572cd43c5b81967 2024-12-07T18:15:51,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/7249527c8c834908b1b3432141f790dd is 50, key is test_row_0/C:col10/1733595349844/Put/seqid=0 2024-12-07T18:15:51,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742323_1499 (size=12151) 2024-12-07T18:15:51,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595411358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:51,368 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:51,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-07T18:15:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-07T18:15:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T18:15:51,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-07T18:15:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:51,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595411708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:51,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595411708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:51,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:51,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595411708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:51,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595411708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:51,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:51,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/7249527c8c834908b1b3432141f790dd 2024-12-07T18:15:51,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/0db967f316be4e42acf3ca2a69865f43 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0db967f316be4e42acf3ca2a69865f43 2024-12-07T18:15:51,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0db967f316be4e42acf3ca2a69865f43, entries=200, sequenceid=167, filesize=14.2 K 2024-12-07T18:15:51,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/217e8202073f4fe2b572cd43c5b81967 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/217e8202073f4fe2b572cd43c5b81967 2024-12-07T18:15:51,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/217e8202073f4fe2b572cd43c5b81967, entries=150, sequenceid=167, filesize=11.9 K 2024-12-07T18:15:51,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/7249527c8c834908b1b3432141f790dd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/7249527c8c834908b1b3432141f790dd 2024-12-07T18:15:51,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/7249527c8c834908b1b3432141f790dd, entries=150, sequenceid=167, filesize=11.9 K 2024-12-07T18:15:51,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0fcb6359e763d0a744acdb75bfd0a652 in 1254ms, sequenceid=167, compaction requested=true 2024-12-07T18:15:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:51,737 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:51,737 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:51,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:51,739 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:51,739 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:51,739 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/B is initiating minor compaction (all files) 2024-12-07T18:15:51,739 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/A is initiating minor compaction (all files) 2024-12-07T18:15:51,739 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/B in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,739 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/A in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,739 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c645ed55f6b84d029688cbfd145f0f5e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/991297f248ac48cfb4da92c558ef8dfe, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/217e8202073f4fe2b572cd43c5b81967] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=35.9 K 2024-12-07T18:15:51,739 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/b5346ec1dcd64476997d18e1c2a5f50c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0bc217077c474963914f8d243268beca, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0db967f316be4e42acf3ca2a69865f43] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=38.2 K 2024-12-07T18:15:51,740 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c645ed55f6b84d029688cbfd145f0f5e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595348279 2024-12-07T18:15:51,740 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting b5346ec1dcd64476997d18e1c2a5f50c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595348279 2024-12-07T18:15:51,740 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bc217077c474963914f8d243268beca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733595348641 2024-12-07T18:15:51,740 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 991297f248ac48cfb4da92c558ef8dfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733595348641 2024-12-07T18:15:51,740 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 0db967f316be4e42acf3ca2a69865f43, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733595349844 2024-12-07T18:15:51,741 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 217e8202073f4fe2b572cd43c5b81967, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733595349844 2024-12-07T18:15:51,748 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#B#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:51,748 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:51,749 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/28213b58eb394bf58352397ac526690d is 50, key is test_row_0/A:col10/1733595349844/Put/seqid=0 2024-12-07T18:15:51,749 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/c8def2fe5d2948b7a2c3c9d3e4bd07e1 is 50, key is test_row_0/B:col10/1733595349844/Put/seqid=0 2024-12-07T18:15:51,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742324_1500 (size=12561) 2024-12-07T18:15:51,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742325_1501 (size=12561) 2024-12-07T18:15:51,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-07T18:15:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:51,849 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T18:15:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:51,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:51,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:51,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:51,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/cf62761f28a244df93036cd12c981049 is 50, key is test_row_0/A:col10/1733595350515/Put/seqid=0 2024-12-07T18:15:51,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742326_1502 (size=12151) 2024-12-07T18:15:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T18:15:52,159 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/28213b58eb394bf58352397ac526690d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/28213b58eb394bf58352397ac526690d 2024-12-07T18:15:52,163 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/c8def2fe5d2948b7a2c3c9d3e4bd07e1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c8def2fe5d2948b7a2c3c9d3e4bd07e1 2024-12-07T18:15:52,164 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/A of 0fcb6359e763d0a744acdb75bfd0a652 into 28213b58eb394bf58352397ac526690d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:52,164 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:52,164 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/A, priority=13, startTime=1733595351737; duration=0sec 2024-12-07T18:15:52,164 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:52,164 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:A 2024-12-07T18:15:52,164 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:52,165 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:52,165 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/C is initiating minor compaction (all files) 2024-12-07T18:15:52,165 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/C in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:52,165 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/90672bd18309430da81ad5f7be066cef, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/4d2dd16cf1da42f7819cb8c5db994cc7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/7249527c8c834908b1b3432141f790dd] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=35.9 K 2024-12-07T18:15:52,166 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 90672bd18309430da81ad5f7be066cef, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733595348279 2024-12-07T18:15:52,166 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d2dd16cf1da42f7819cb8c5db994cc7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733595348641 2024-12-07T18:15:52,166 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7249527c8c834908b1b3432141f790dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733595349844 2024-12-07T18:15:52,167 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/B of 0fcb6359e763d0a744acdb75bfd0a652 into c8def2fe5d2948b7a2c3c9d3e4bd07e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:52,167 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:52,167 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/B, priority=13, startTime=1733595351737; duration=0sec 2024-12-07T18:15:52,168 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:52,168 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:B 2024-12-07T18:15:52,174 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#C#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:52,175 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bbc147d50a224e6bbc7a0a0a7b0a23a4 is 50, key is test_row_0/C:col10/1733595349844/Put/seqid=0 2024-12-07T18:15:52,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742327_1503 (size=12561) 2024-12-07T18:15:52,186 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bbc147d50a224e6bbc7a0a0a7b0a23a4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bbc147d50a224e6bbc7a0a0a7b0a23a4 2024-12-07T18:15:52,191 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/C of 0fcb6359e763d0a744acdb75bfd0a652 into bbc147d50a224e6bbc7a0a0a7b0a23a4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:52,191 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:52,191 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/C, priority=13, startTime=1733595351737; duration=0sec 2024-12-07T18:15:52,191 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:52,191 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:C 2024-12-07T18:15:52,258 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/cf62761f28a244df93036cd12c981049 2024-12-07T18:15:52,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/fd9e3ee92f344a5a9763a697e300377f is 50, key is test_row_0/B:col10/1733595350515/Put/seqid=0 2024-12-07T18:15:52,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742328_1504 (size=12151) 2024-12-07T18:15:52,674 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/fd9e3ee92f344a5a9763a697e300377f 2024-12-07T18:15:52,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/a006e5ec3d574c179aaa818a1e5d128a is 50, key is test_row_0/C:col10/1733595350515/Put/seqid=0 2024-12-07T18:15:52,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742329_1505 (size=12151) 2024-12-07T18:15:52,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:52,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:52,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:52,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595412754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:52,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595412754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595412755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:52,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595412765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:52,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:52,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:52,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595412869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,876 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:52,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:52,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595412878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595412878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:52,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595412878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:52,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T18:15:53,085 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/a006e5ec3d574c179aaa818a1e5d128a 2024-12-07T18:15:53,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/cf62761f28a244df93036cd12c981049 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/cf62761f28a244df93036cd12c981049 2024-12-07T18:15:53,091 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/cf62761f28a244df93036cd12c981049, entries=150, sequenceid=182, filesize=11.9 K 2024-12-07T18:15:53,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/fd9e3ee92f344a5a9763a697e300377f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/fd9e3ee92f344a5a9763a697e300377f 2024-12-07T18:15:53,095 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/fd9e3ee92f344a5a9763a697e300377f, entries=150, sequenceid=182, filesize=11.9 K 2024-12-07T18:15:53,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/a006e5ec3d574c179aaa818a1e5d128a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a006e5ec3d574c179aaa818a1e5d128a 2024-12-07T18:15:53,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595413088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,100 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a006e5ec3d574c179aaa818a1e5d128a, entries=150, sequenceid=182, filesize=11.9 K 2024-12-07T18:15:53,101 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0fcb6359e763d0a744acdb75bfd0a652 in 1252ms, sequenceid=182, compaction requested=false 2024-12-07T18:15:53,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:53,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:53,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-07T18:15:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-07T18:15:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:53,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T18:15:53,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-07T18:15:53,104 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1700 sec 2024-12-07T18:15:53,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:53,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:53,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:53,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:53,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:53,105 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:53,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.1760 sec 2024-12-07T18:15:53,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/6c825ba98b5948d4a44f27f18461cd3c is 50, key is test_row_0/A:col10/1733595353103/Put/seqid=0 2024-12-07T18:15:53,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742330_1506 (size=16931) 2024-12-07T18:15:53,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595413136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595413137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595413143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595413258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595413258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595413258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595413408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595413468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595413468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595413468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/6c825ba98b5948d4a44f27f18461cd3c 2024-12-07T18:15:53,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/8248911b25034e4093e3eee62a32b9dd is 50, key is test_row_0/B:col10/1733595353103/Put/seqid=0 2024-12-07T18:15:53,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742331_1507 (size=12151) 2024-12-07T18:15:53,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595413789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595413789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595413789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/8248911b25034e4093e3eee62a32b9dd 2024-12-07T18:15:53,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595413928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:53,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:53,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:53,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bf5e8a1e1a0843f487131de9f35a9f8b is 50, key is test_row_0/C:col10/1733595353103/Put/seqid=0 2024-12-07T18:15:53,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742332_1508 (size=12151) 2024-12-07T18:15:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595414308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595414308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:54,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:54,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595414308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:54,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bf5e8a1e1a0843f487131de9f35a9f8b 2024-12-07T18:15:54,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/6c825ba98b5948d4a44f27f18461cd3c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/6c825ba98b5948d4a44f27f18461cd3c 2024-12-07T18:15:54,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/6c825ba98b5948d4a44f27f18461cd3c, entries=250, sequenceid=204, filesize=16.5 K 2024-12-07T18:15:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/8248911b25034e4093e3eee62a32b9dd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/8248911b25034e4093e3eee62a32b9dd 2024-12-07T18:15:54,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/8248911b25034e4093e3eee62a32b9dd, entries=150, sequenceid=204, filesize=11.9 K 2024-12-07T18:15:54,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bf5e8a1e1a0843f487131de9f35a9f8b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf5e8a1e1a0843f487131de9f35a9f8b 2024-12-07T18:15:54,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf5e8a1e1a0843f487131de9f35a9f8b, entries=150, sequenceid=204, filesize=11.9 K 2024-12-07T18:15:54,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 0fcb6359e763d0a744acdb75bfd0a652 in 1261ms, sequenceid=204, compaction requested=true 2024-12-07T18:15:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:54,365 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:54,365 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:54,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:54,366 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:54,366 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/A is initiating minor compaction (all files) 2024-12-07T18:15:54,366 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/A in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:54,366 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/28213b58eb394bf58352397ac526690d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/cf62761f28a244df93036cd12c981049, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/6c825ba98b5948d4a44f27f18461cd3c] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=40.7 K 2024-12-07T18:15:54,366 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:54,366 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/B is initiating minor compaction (all files) 2024-12-07T18:15:54,366 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/B in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:54,366 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c8def2fe5d2948b7a2c3c9d3e4bd07e1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/fd9e3ee92f344a5a9763a697e300377f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/8248911b25034e4093e3eee62a32b9dd] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=36.0 K 2024-12-07T18:15:54,367 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting c8def2fe5d2948b7a2c3c9d3e4bd07e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733595349844 2024-12-07T18:15:54,367 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28213b58eb394bf58352397ac526690d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733595349844 2024-12-07T18:15:54,367 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting fd9e3ee92f344a5a9763a697e300377f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733595350515 2024-12-07T18:15:54,367 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf62761f28a244df93036cd12c981049, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733595350515 2024-12-07T18:15:54,367 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 8248911b25034e4093e3eee62a32b9dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733595352754 2024-12-07T18:15:54,367 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c825ba98b5948d4a44f27f18461cd3c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733595352752 2024-12-07T18:15:54,376 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:54,377 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/7df8267fae9548fe817f44e58f4c6cbb is 50, key is test_row_0/B:col10/1733595353103/Put/seqid=0 2024-12-07T18:15:54,377 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#A#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:54,378 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/c1ba5dfca91649c48e15d57f2067939a is 50, key is test_row_0/A:col10/1733595353103/Put/seqid=0 2024-12-07T18:15:54,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742333_1509 (size=12663) 2024-12-07T18:15:54,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742334_1510 (size=12663) 2024-12-07T18:15:54,404 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/c1ba5dfca91649c48e15d57f2067939a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c1ba5dfca91649c48e15d57f2067939a 2024-12-07T18:15:54,408 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/A of 0fcb6359e763d0a744acdb75bfd0a652 into c1ba5dfca91649c48e15d57f2067939a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:54,408 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:54,408 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/A, priority=13, startTime=1733595354365; duration=0sec 2024-12-07T18:15:54,408 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:54,408 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:A 2024-12-07T18:15:54,408 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:15:54,410 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:15:54,410 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/C is initiating minor compaction (all files) 2024-12-07T18:15:54,410 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/C in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:54,410 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bbc147d50a224e6bbc7a0a0a7b0a23a4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a006e5ec3d574c179aaa818a1e5d128a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf5e8a1e1a0843f487131de9f35a9f8b] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=36.0 K 2024-12-07T18:15:54,410 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbc147d50a224e6bbc7a0a0a7b0a23a4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733595349844 2024-12-07T18:15:54,411 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting a006e5ec3d574c179aaa818a1e5d128a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733595350515 2024-12-07T18:15:54,411 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf5e8a1e1a0843f487131de9f35a9f8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733595352754 2024-12-07T18:15:54,416 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#C#compaction#431 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:54,417 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/f7d8dc2094ba4ad7b843c68f1be8493d is 50, key is test_row_0/C:col10/1733595353103/Put/seqid=0 2024-12-07T18:15:54,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742335_1511 (size=12663) 2024-12-07T18:15:54,801 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/7df8267fae9548fe817f44e58f4c6cbb as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7df8267fae9548fe817f44e58f4c6cbb 2024-12-07T18:15:54,805 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/B of 0fcb6359e763d0a744acdb75bfd0a652 into 7df8267fae9548fe817f44e58f4c6cbb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:54,805 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:54,805 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/B, priority=13, startTime=1733595354365; duration=0sec 2024-12-07T18:15:54,805 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:54,805 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:B 2024-12-07T18:15:54,824 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/f7d8dc2094ba4ad7b843c68f1be8493d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f7d8dc2094ba4ad7b843c68f1be8493d 2024-12-07T18:15:54,828 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/C of 0fcb6359e763d0a744acdb75bfd0a652 into f7d8dc2094ba4ad7b843c68f1be8493d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:54,828 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:54,828 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/C, priority=13, startTime=1733595354365; duration=0sec 2024-12-07T18:15:54,828 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:54,828 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:C 2024-12-07T18:15:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:54,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T18:15:54,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:54,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:54,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:54,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:54,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:54,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:54,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/44745b3aa2c94dd19dc250762b49f15e is 50, key is test_row_0/A:col10/1733595353135/Put/seqid=0 2024-12-07T18:15:54,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742336_1512 (size=14541) 2024-12-07T18:15:55,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595415066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:55,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-07T18:15:55,089 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:55,089 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-07T18:15:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T18:15:55,090 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:55,091 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:55,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:55,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595415188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:55,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T18:15:55,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-07T18:15:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:55,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595415328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:55,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595415329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:55,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595415329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:55,336 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,337 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:55,337 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_0' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:55,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/44745b3aa2c94dd19dc250762b49f15e 2024-12-07T18:15:55,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/adef8eccc20a4273843b3c98d98d91cb is 50, key is test_row_0/B:col10/1733595353135/Put/seqid=0 2024-12-07T18:15:55,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742337_1513 (size=12151) 2024-12-07T18:15:55,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/adef8eccc20a4273843b3c98d98d91cb 2024-12-07T18:15:55,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bf83d3cae982472da3429aec96e773f4 is 50, key is test_row_0/C:col10/1733595353135/Put/seqid=0 2024-12-07T18:15:55,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742338_1514 (size=12151) 2024-12-07T18:15:55,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-07T18:15:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:55,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T18:15:55,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54654 deadline: 1733595415408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:55,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:55,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595415409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:55,419 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:55,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-07T18:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:55,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-07T18:15:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:55,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T18:15:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595415728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:55,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bf83d3cae982472da3429aec96e773f4 2024-12-07T18:15:55,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/44745b3aa2c94dd19dc250762b49f15e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/44745b3aa2c94dd19dc250762b49f15e 2024-12-07T18:15:55,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/44745b3aa2c94dd19dc250762b49f15e, entries=200, sequenceid=221, filesize=14.2 K 2024-12-07T18:15:55,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/adef8eccc20a4273843b3c98d98d91cb as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/adef8eccc20a4273843b3c98d98d91cb 2024-12-07T18:15:55,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/adef8eccc20a4273843b3c98d98d91cb, entries=150, sequenceid=221, filesize=11.9 K 2024-12-07T18:15:55,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bf83d3cae982472da3429aec96e773f4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf83d3cae982472da3429aec96e773f4 2024-12-07T18:15:55,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf83d3cae982472da3429aec96e773f4, entries=150, sequenceid=221, filesize=11.9 K 2024-12-07T18:15:55,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 0fcb6359e763d0a744acdb75bfd0a652 in 860ms, sequenceid=221, compaction requested=false 2024-12-07T18:15:55,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:55,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-07T18:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:55,855 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T18:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:55,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/08e003dd58724b50b70a909609c205ee is 50, key is test_row_0/A:col10/1733595355020/Put/seqid=0 2024-12-07T18:15:55,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742339_1515 (size=12151) 2024-12-07T18:15:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T18:15:56,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:56,265 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/08e003dd58724b50b70a909609c205ee 2024-12-07T18:15:56,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/41a5af55d81f44caab3a36801da9e5b3 is 50, key is test_row_0/B:col10/1733595355020/Put/seqid=0 2024-12-07T18:15:56,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742340_1516 (size=12151) 2024-12-07T18:15:56,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595416341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:56,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595416458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:56,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595416668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:56,677 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/41a5af55d81f44caab3a36801da9e5b3 2024-12-07T18:15:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/ebfc77d500a74634b7f53e4c01c0aa8e is 50, key is test_row_0/C:col10/1733595355020/Put/seqid=0 2024-12-07T18:15:56,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742341_1517 (size=12151) 2024-12-07T18:15:56,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595416978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,088 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/ebfc77d500a74634b7f53e4c01c0aa8e 2024-12-07T18:15:57,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/08e003dd58724b50b70a909609c205ee as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/08e003dd58724b50b70a909609c205ee 2024-12-07T18:15:57,094 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/08e003dd58724b50b70a909609c205ee, entries=150, sequenceid=239, filesize=11.9 K 2024-12-07T18:15:57,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/41a5af55d81f44caab3a36801da9e5b3 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/41a5af55d81f44caab3a36801da9e5b3 2024-12-07T18:15:57,099 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/41a5af55d81f44caab3a36801da9e5b3, entries=150, sequenceid=239, filesize=11.9 K 2024-12-07T18:15:57,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/ebfc77d500a74634b7f53e4c01c0aa8e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebfc77d500a74634b7f53e4c01c0aa8e 2024-12-07T18:15:57,106 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebfc77d500a74634b7f53e4c01c0aa8e, entries=150, sequenceid=239, filesize=11.9 K 2024-12-07T18:15:57,107 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 0fcb6359e763d0a744acdb75bfd0a652 in 1252ms, sequenceid=239, compaction requested=true 2024-12-07T18:15:57,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:57,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-07T18:15:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-07T18:15:57,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-07T18:15:57,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0180 sec 2024-12-07T18:15:57,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.0230 sec 2024-12-07T18:15:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-07T18:15:57,239 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:57,239 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-07T18:15:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T18:15:57,241 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:57,242 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:57,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T18:15:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:57,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T18:15:57,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:57,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:57,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:57,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:57,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:57,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:57,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/8bafc920bfe347328c05ceb4ee6f0b6b is 50, key is test_row_0/A:col10/1733595357354/Put/seqid=0 2024-12-07T18:15:57,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742342_1518 (size=14541) 2024-12-07T18:15:57,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/8bafc920bfe347328c05ceb4ee6f0b6b 2024-12-07T18:15:57,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/caebecd378e64a86b62745101d801153 is 50, key is test_row_0/B:col10/1733595357354/Put/seqid=0 2024-12-07T18:15:57,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742343_1519 (size=12151) 2024-12-07T18:15:57,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/caebecd378e64a86b62745101d801153 2024-12-07T18:15:57,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bdcb21daba46435e85b58bdc9873ab0b is 50, key is test_row_0/C:col10/1733595357354/Put/seqid=0 2024-12-07T18:15:57,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742344_1520 (size=12151) 2024-12-07T18:15:57,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T18:15:57,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:57,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:57,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:57,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595417396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:57,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595417406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595417412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595417488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595417518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:57,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595417519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595417528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T18:15:57,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:57,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:57,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T18:15:57,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T18:15:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:57,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:57,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595417728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:57,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595417729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:57,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595417748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:57,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:57,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bdcb21daba46435e85b58bdc9873ab0b 2024-12-07T18:15:57,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/8bafc920bfe347328c05ceb4ee6f0b6b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/8bafc920bfe347328c05ceb4ee6f0b6b 2024-12-07T18:15:57,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/8bafc920bfe347328c05ceb4ee6f0b6b, entries=200, sequenceid=254, filesize=14.2 K 2024-12-07T18:15:57,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/caebecd378e64a86b62745101d801153 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/caebecd378e64a86b62745101d801153 2024-12-07T18:15:57,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/caebecd378e64a86b62745101d801153, entries=150, sequenceid=254, filesize=11.9 K 2024-12-07T18:15:57,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bdcb21daba46435e85b58bdc9873ab0b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bdcb21daba46435e85b58bdc9873ab0b 2024-12-07T18:15:57,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bdcb21daba46435e85b58bdc9873ab0b, entries=150, sequenceid=254, filesize=11.9 K 2024-12-07T18:15:57,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 0fcb6359e763d0a744acdb75bfd0a652 in 443ms, sequenceid=254, compaction requested=true 2024-12-07T18:15:57,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:57,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:15:57,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:57,799 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:15:57,799 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:15:57,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:15:57,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:57,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:15:57,800 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53896 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:15:57,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:57,800 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/A is initiating minor compaction (all files) 2024-12-07T18:15:57,800 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/A in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,800 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c1ba5dfca91649c48e15d57f2067939a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/44745b3aa2c94dd19dc250762b49f15e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/08e003dd58724b50b70a909609c205ee, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/8bafc920bfe347328c05ceb4ee6f0b6b] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=52.6 K 2024-12-07T18:15:57,801 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1ba5dfca91649c48e15d57f2067939a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733595352754 2024-12-07T18:15:57,801 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44745b3aa2c94dd19dc250762b49f15e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733595353117 2024-12-07T18:15:57,801 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:15:57,802 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/B is initiating minor compaction (all files) 2024-12-07T18:15:57,802 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/B in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,802 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7df8267fae9548fe817f44e58f4c6cbb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/adef8eccc20a4273843b3c98d98d91cb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/41a5af55d81f44caab3a36801da9e5b3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/caebecd378e64a86b62745101d801153] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=48.0 K 2024-12-07T18:15:57,802 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08e003dd58724b50b70a909609c205ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733595355020 2024-12-07T18:15:57,802 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df8267fae9548fe817f44e58f4c6cbb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733595352754 2024-12-07T18:15:57,802 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bafc920bfe347328c05ceb4ee6f0b6b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733595356294 2024-12-07T18:15:57,803 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting adef8eccc20a4273843b3c98d98d91cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733595353117 2024-12-07T18:15:57,803 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a5af55d81f44caab3a36801da9e5b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733595355020 2024-12-07T18:15:57,804 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting caebecd378e64a86b62745101d801153, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733595356331 2024-12-07T18:15:57,812 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#A#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:57,813 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/60f4bf4da411411b9fb2b3655f177814 is 50, key is test_row_0/A:col10/1733595357354/Put/seqid=0 2024-12-07T18:15:57,816 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#B#compaction#442 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:57,816 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/d56b70920aac4574885c27eb03c7904e is 50, key is test_row_0/B:col10/1733595357354/Put/seqid=0 2024-12-07T18:15:57,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742346_1522 (size=12799) 2024-12-07T18:15:57,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742345_1521 (size=12799) 2024-12-07T18:15:57,833 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/60f4bf4da411411b9fb2b3655f177814 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/60f4bf4da411411b9fb2b3655f177814 2024-12-07T18:15:57,839 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/A of 0fcb6359e763d0a744acdb75bfd0a652 into 60f4bf4da411411b9fb2b3655f177814(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:57,839 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:57,839 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/A, priority=12, startTime=1733595357799; duration=0sec 2024-12-07T18:15:57,839 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:15:57,839 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:A 2024-12-07T18:15:57,840 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:15:57,841 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:15:57,841 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/C is initiating minor compaction (all files) 2024-12-07T18:15:57,841 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/C in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,841 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f7d8dc2094ba4ad7b843c68f1be8493d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf83d3cae982472da3429aec96e773f4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebfc77d500a74634b7f53e4c01c0aa8e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bdcb21daba46435e85b58bdc9873ab0b] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=48.0 K 2024-12-07T18:15:57,842 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7d8dc2094ba4ad7b843c68f1be8493d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733595352754 2024-12-07T18:15:57,842 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf83d3cae982472da3429aec96e773f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733595353117 2024-12-07T18:15:57,842 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebfc77d500a74634b7f53e4c01c0aa8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1733595355020 2024-12-07T18:15:57,843 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdcb21daba46435e85b58bdc9873ab0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733595356331 2024-12-07T18:15:57,852 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#C#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:15:57,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-07T18:15:57,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:57,853 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/2f0d938586ba41958b15716555c1b223 is 50, key is test_row_0/C:col10/1733595357354/Put/seqid=0 2024-12-07T18:15:57,853 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-07T18:15:57,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:57,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:57,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:57,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:57,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:57,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:57,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742347_1523 (size=12799) 2024-12-07T18:15:57,863 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/2f0d938586ba41958b15716555c1b223 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/2f0d938586ba41958b15716555c1b223 2024-12-07T18:15:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/51ab9407b7444b6cbeef75522b27af39 is 50, key is test_row_0/A:col10/1733595357393/Put/seqid=0 2024-12-07T18:15:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T18:15:57,871 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/C of 0fcb6359e763d0a744acdb75bfd0a652 into 2f0d938586ba41958b15716555c1b223(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:57,871 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:57,871 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/C, priority=12, startTime=1733595357800; duration=0sec 2024-12-07T18:15:57,871 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:57,871 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:C 2024-12-07T18:15:57,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742348_1524 (size=12301) 2024-12-07T18:15:58,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:58,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:58,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595418083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595418087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595418090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595418198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595418208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595418208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,232 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/d56b70920aac4574885c27eb03c7904e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/d56b70920aac4574885c27eb03c7904e 2024-12-07T18:15:58,237 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/B of 0fcb6359e763d0a744acdb75bfd0a652 into d56b70920aac4574885c27eb03c7904e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:15:58,237 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:58,237 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/B, priority=12, startTime=1733595357799; duration=0sec 2024-12-07T18:15:58,237 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:15:58,237 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:B 2024-12-07T18:15:58,274 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/51ab9407b7444b6cbeef75522b27af39 2024-12-07T18:15:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/a7082d7a1f85420bb31256d4b0025acc is 50, key is test_row_0/B:col10/1733595357393/Put/seqid=0 2024-12-07T18:15:58,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742349_1525 (size=12301) 2024-12-07T18:15:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T18:15:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595418409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595418428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595418428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,434 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595418508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,516 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:15:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,516 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,684 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/a7082d7a1f85420bb31256d4b0025acc 2024-12-07T18:15:58,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/0d11818bffe74845858823f0daf70427 is 50, key is test_row_0/C:col10/1733595357393/Put/seqid=0 2024-12-07T18:15:58,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742350_1526 (size=12301) 2024-12-07T18:15:58,720 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/0d11818bffe74845858823f0daf70427 2024-12-07T18:15:58,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/51ab9407b7444b6cbeef75522b27af39 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/51ab9407b7444b6cbeef75522b27af39 2024-12-07T18:15:58,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595418730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,740 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/51ab9407b7444b6cbeef75522b27af39, entries=150, sequenceid=272, filesize=12.0 K 2024-12-07T18:15:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/a7082d7a1f85420bb31256d4b0025acc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/a7082d7a1f85420bb31256d4b0025acc 2024-12-07T18:15:58,745 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/a7082d7a1f85420bb31256d4b0025acc, entries=150, sequenceid=272, filesize=12.0 K 2024-12-07T18:15:58,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595418738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:58,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/0d11818bffe74845858823f0daf70427 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0d11818bffe74845858823f0daf70427 2024-12-07T18:15:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595418738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:58,756 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0d11818bffe74845858823f0daf70427, entries=150, sequenceid=272, filesize=12.0 K 2024-12-07T18:15:58,757 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0fcb6359e763d0a744acdb75bfd0a652 in 904ms, sequenceid=272, compaction requested=false 2024-12-07T18:15:58,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:15:58,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:58,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-07T18:15:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-07T18:15:58,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-07T18:15:58,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5180 sec 2024-12-07T18:15:58,764 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.5240 sec 2024-12-07T18:15:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:15:59,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T18:15:59,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:15:59,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:59,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:15:59,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:59,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:15:59,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:15:59,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/0a9cfb5dbaa9468f82970db8e970a5ac is 50, key is test_row_0/A:col10/1733595358074/Put/seqid=0 2024-12-07T18:15:59,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742351_1527 (size=14741) 2024-12-07T18:15:59,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595419298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595419301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595419308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-07T18:15:59,389 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:15:59,389 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:15:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-07T18:15:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T18:15:59,391 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:15:59,391 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:15:59,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:15:59,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595419418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595419418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595419418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T18:15:59,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:15:59,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:59,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:59,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:59,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:59,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:59,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595419638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595419638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595419638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/0a9cfb5dbaa9468f82970db8e970a5ac 2024-12-07T18:15:59,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/99fad172cc2d424b9cec18b27bc3376c is 50, key is test_row_0/B:col10/1733595358074/Put/seqid=0 2024-12-07T18:15:59,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742352_1528 (size=12301) 2024-12-07T18:15:59,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:15:59,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:59,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:59,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:59,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T18:15:59,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:15:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:15:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:15:59,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:59,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:15:59,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595419958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595419959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:15:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595419959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:15:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:15:59,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:00,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:00,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:00,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T18:16:00,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/99fad172cc2d424b9cec18b27bc3376c 2024-12-07T18:16:00,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/d7cf0ecc53ad4e32a9db956a7f027335 is 50, key is test_row_0/C:col10/1733595358074/Put/seqid=0 2024-12-07T18:16:00,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742353_1529 (size=12301) 2024-12-07T18:16:00,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:00,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:00,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:00,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:00,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595420468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:00,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:00,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:00,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:00,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/d7cf0ecc53ad4e32a9db956a7f027335 2024-12-07T18:16:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595420478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595420478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:00,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/0a9cfb5dbaa9468f82970db8e970a5ac as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0a9cfb5dbaa9468f82970db8e970a5ac 2024-12-07T18:16:00,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0a9cfb5dbaa9468f82970db8e970a5ac, entries=200, sequenceid=290, filesize=14.4 K 2024-12-07T18:16:00,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/99fad172cc2d424b9cec18b27bc3376c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/99fad172cc2d424b9cec18b27bc3376c 2024-12-07T18:16:00,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/99fad172cc2d424b9cec18b27bc3376c, entries=150, sequenceid=290, filesize=12.0 K 2024-12-07T18:16:00,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/d7cf0ecc53ad4e32a9db956a7f027335 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/d7cf0ecc53ad4e32a9db956a7f027335 2024-12-07T18:16:00,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/d7cf0ecc53ad4e32a9db956a7f027335, entries=150, sequenceid=290, filesize=12.0 K 2024-12-07T18:16:00,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0fcb6359e763d0a744acdb75bfd0a652 in 1245ms, sequenceid=290, compaction requested=true 2024-12-07T18:16:00,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:16:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:16:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:16:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:00,500 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:16:00,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:00,500 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:00,502 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39841 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:00,502 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:00,502 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/B is initiating minor compaction (all files) 2024-12-07T18:16:00,502 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/A is initiating minor compaction (all files) 2024-12-07T18:16:00,503 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/B in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,503 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/A in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,503 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/d56b70920aac4574885c27eb03c7904e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/a7082d7a1f85420bb31256d4b0025acc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/99fad172cc2d424b9cec18b27bc3376c] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=36.5 K 2024-12-07T18:16:00,503 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/60f4bf4da411411b9fb2b3655f177814, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/51ab9407b7444b6cbeef75522b27af39, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0a9cfb5dbaa9468f82970db8e970a5ac] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=38.9 K 2024-12-07T18:16:00,503 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting d56b70920aac4574885c27eb03c7904e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733595356331 2024-12-07T18:16:00,503 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60f4bf4da411411b9fb2b3655f177814, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733595356331 2024-12-07T18:16:00,503 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51ab9407b7444b6cbeef75522b27af39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733595357391 2024-12-07T18:16:00,503 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting a7082d7a1f85420bb31256d4b0025acc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733595357391 2024-12-07T18:16:00,504 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 99fad172cc2d424b9cec18b27bc3376c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733595358074 2024-12-07T18:16:00,504 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a9cfb5dbaa9468f82970db8e970a5ac, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733595358074 2024-12-07T18:16:00,510 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#A#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:00,511 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#B#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:00,511 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/d3114f2bfecc47d38f46bcfb9b4ffcf2 is 50, key is test_row_0/A:col10/1733595358074/Put/seqid=0 2024-12-07T18:16:00,511 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/49f364b75b774d45abd6d334f617652f is 50, key is test_row_0/B:col10/1733595358074/Put/seqid=0 2024-12-07T18:16:00,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742355_1531 (size=13051) 2024-12-07T18:16:00,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742354_1530 (size=13051) 2024-12-07T18:16:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T18:16:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:16:00,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-07T18:16:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:16:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:16:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:16:00,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:00,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/ba156387961e4cf2b207b0b18003568a is 50, key is test_row_0/A:col10/1733595359306/Put/seqid=0 2024-12-07T18:16:00,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742356_1532 (size=14741) 2024-12-07T18:16:00,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:00,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:00,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595420664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:00,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:00,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595420788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:00,799 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:00,800 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:00,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:00,922 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/49f364b75b774d45abd6d334f617652f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/49f364b75b774d45abd6d334f617652f 2024-12-07T18:16:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:00,928 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/B of 0fcb6359e763d0a744acdb75bfd0a652 into 49f364b75b774d45abd6d334f617652f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:00,928 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:16:00,928 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/B, priority=13, startTime=1733595360500; duration=0sec 2024-12-07T18:16:00,928 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:00,928 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:B 2024-12-07T18:16:00,928 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:00,929 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:00,929 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/C is initiating minor compaction (all files) 2024-12-07T18:16:00,929 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/C in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:00,929 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/2f0d938586ba41958b15716555c1b223, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0d11818bffe74845858823f0daf70427, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/d7cf0ecc53ad4e32a9db956a7f027335] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=36.5 K 2024-12-07T18:16:00,929 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/d3114f2bfecc47d38f46bcfb9b4ffcf2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/d3114f2bfecc47d38f46bcfb9b4ffcf2 2024-12-07T18:16:00,930 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f0d938586ba41958b15716555c1b223, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733595356331 2024-12-07T18:16:00,930 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d11818bffe74845858823f0daf70427, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1733595357391 2024-12-07T18:16:00,931 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting d7cf0ecc53ad4e32a9db956a7f027335, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733595358074 2024-12-07T18:16:00,934 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/A of 0fcb6359e763d0a744acdb75bfd0a652 into d3114f2bfecc47d38f46bcfb9b4ffcf2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:00,934 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:16:00,934 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/A, priority=13, startTime=1733595360500; duration=0sec 2024-12-07T18:16:00,934 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:00,934 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:A 2024-12-07T18:16:00,946 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#C#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:00,947 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bf4cb87245424b8ea778ab37a0bb59ee is 50, key is test_row_0/C:col10/1733595358074/Put/seqid=0 2024-12-07T18:16:00,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/ba156387961e4cf2b207b0b18003568a 2024-12-07T18:16:00,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742357_1533 (size=13051) 2024-12-07T18:16:00,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/7daa09915ba746b49dd35557033c71f8 is 50, key is test_row_0/B:col10/1733595359306/Put/seqid=0 2024-12-07T18:16:00,963 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/bf4cb87245424b8ea778ab37a0bb59ee as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf4cb87245424b8ea778ab37a0bb59ee 2024-12-07T18:16:00,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742358_1534 (size=12301) 2024-12-07T18:16:00,967 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/C of 0fcb6359e763d0a744acdb75bfd0a652 into bf4cb87245424b8ea778ab37a0bb59ee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:00,967 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:16:00,967 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/C, priority=13, startTime=1733595360500; duration=0sec 2024-12-07T18:16:00,967 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:00,968 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:C 2024-12-07T18:16:00,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/7daa09915ba746b49dd35557033c71f8 2024-12-07T18:16:00,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/909480df338149a9a8f3b4c7400fc983 is 50, key is test_row_0/C:col10/1733595359306/Put/seqid=0 2024-12-07T18:16:00,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742359_1535 (size=12301) 2024-12-07T18:16:00,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/909480df338149a9a8f3b4c7400fc983 2024-12-07T18:16:00,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/ba156387961e4cf2b207b0b18003568a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/ba156387961e4cf2b207b0b18003568a 2024-12-07T18:16:00,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/ba156387961e4cf2b207b0b18003568a, entries=200, sequenceid=307, filesize=14.4 K 2024-12-07T18:16:00,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/7daa09915ba746b49dd35557033c71f8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7daa09915ba746b49dd35557033c71f8 2024-12-07T18:16:00,991 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7daa09915ba746b49dd35557033c71f8, entries=150, sequenceid=307, filesize=12.0 K 2024-12-07T18:16:00,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/909480df338149a9a8f3b4c7400fc983 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/909480df338149a9a8f3b4c7400fc983 2024-12-07T18:16:00,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/909480df338149a9a8f3b4c7400fc983, entries=150, sequenceid=307, filesize=12.0 K 2024-12-07T18:16:00,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 0fcb6359e763d0a744acdb75bfd0a652 in 454ms, sequenceid=307, compaction requested=false 2024-12-07T18:16:00,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:16:01,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:16:01,015 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T18:16:01,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:16:01,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:01,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:16:01,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:01,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:16:01,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:01,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/aff9cb16a8c443db977d8ad4b0a90068 is 50, key is test_row_0/A:col10/1733595360663/Put/seqid=0 2024-12-07T18:16:01,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742360_1536 (size=14741) 2024-12-07T18:16:01,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595421138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:01,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595421258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,266 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:01,326 INFO [Thread-2083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2083. 2024-12-07T18:16:01,326 DEBUG [Thread-2083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:01,326 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:01,326 DEBUG [Thread-2083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:01,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:01,327 INFO [Thread-2085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2085. 2024-12-07T18:16:01,327 DEBUG [Thread-2085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:01,327 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:01,327 DEBUG [Thread-2085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:01,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:01,334 INFO [Thread-2081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2081. 2024-12-07T18:16:01,334 DEBUG [Thread-2081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:01,334 DEBUG [Thread-2081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:01,334 DEBUG [Thread-2081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:01,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:01,337 INFO [Thread-2087 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2087. 2024-12-07T18:16:01,337 DEBUG [Thread-2087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:01,337 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:01,338 DEBUG [Thread-2087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:01,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:01,338 INFO [Thread-2089 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2089. 2024-12-07T18:16:01,338 DEBUG [Thread-2089 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:01,338 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:01,338 DEBUG [Thread-2089 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:01,338 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:01,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/aff9cb16a8c443db977d8ad4b0a90068 2024-12-07T18:16:01,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/ba61e628efe543809b13f332ad6b855f is 50, key is test_row_0/B:col10/1733595360663/Put/seqid=0 2024-12-07T18:16:01,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742361_1537 (size=12301) 2024-12-07T18:16:01,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595421479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:01,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1733595421488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:01,489 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54650 deadline: 1733595421498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1733595421498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:01,499 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:01,499 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in 0fcb6359e763d0a744acdb75bfd0a652 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:01,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T18:16:01,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:01,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:01,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:01,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54678 deadline: 1733595421788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 , the old value is region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., hostname=92acd7e2389b,33947,1733595217075, seqNum=2 because the exception is null or not the one we care about 2024-12-07T18:16:01,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/ba61e628efe543809b13f332ad6b855f 2024-12-07T18:16:01,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/f0c7113411a5437a9efb48c8cb99a955 is 50, key is test_row_0/C:col10/1733595360663/Put/seqid=0 2024-12-07T18:16:01,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742362_1538 (size=12301) 2024-12-07T18:16:01,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:01,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:01,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:01,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:02,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:02,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:02,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:02,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:02,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:02,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:02,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/f0c7113411a5437a9efb48c8cb99a955 2024-12-07T18:16:02,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/aff9cb16a8c443db977d8ad4b0a90068 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/aff9cb16a8c443db977d8ad4b0a90068 2024-12-07T18:16:02,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/aff9cb16a8c443db977d8ad4b0a90068, entries=200, sequenceid=326, filesize=14.4 K 2024-12-07T18:16:02,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/ba61e628efe543809b13f332ad6b855f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/ba61e628efe543809b13f332ad6b855f 2024-12-07T18:16:02,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/ba61e628efe543809b13f332ad6b855f, entries=150, sequenceid=326, filesize=12.0 K 2024-12-07T18:16:02,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/f0c7113411a5437a9efb48c8cb99a955 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f0c7113411a5437a9efb48c8cb99a955 2024-12-07T18:16:02,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f0c7113411a5437a9efb48c8cb99a955, entries=150, sequenceid=326, filesize=12.0 K 2024-12-07T18:16:02,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 0fcb6359e763d0a744acdb75bfd0a652 in 1239ms, sequenceid=326, compaction requested=true 2024-12-07T18:16:02,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:16:02,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:16:02,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:02,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:16:02,255 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:02,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:02,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0fcb6359e763d0a744acdb75bfd0a652:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:16:02,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:02,255 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:02,255 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:02,255 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:02,255 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/A is initiating minor compaction (all files) 2024-12-07T18:16:02,255 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/B is initiating minor compaction (all files) 2024-12-07T18:16:02,255 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/A in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:02,255 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/B in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:02,256 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/d3114f2bfecc47d38f46bcfb9b4ffcf2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/ba156387961e4cf2b207b0b18003568a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/aff9cb16a8c443db977d8ad4b0a90068] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=41.5 K 2024-12-07T18:16:02,256 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/49f364b75b774d45abd6d334f617652f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7daa09915ba746b49dd35557033c71f8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/ba61e628efe543809b13f332ad6b855f] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=36.8 K 2024-12-07T18:16:02,256 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f364b75b774d45abd6d334f617652f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733595358074 2024-12-07T18:16:02,256 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3114f2bfecc47d38f46bcfb9b4ffcf2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733595358074 2024-12-07T18:16:02,256 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba156387961e4cf2b207b0b18003568a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733595359296 2024-12-07T18:16:02,256 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7daa09915ba746b49dd35557033c71f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733595359296 2024-12-07T18:16:02,256 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting aff9cb16a8c443db977d8ad4b0a90068, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733595360608 2024-12-07T18:16:02,256 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting ba61e628efe543809b13f332ad6b855f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733595360650 2024-12-07T18:16:02,262 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:02,262 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#A#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:02,262 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/9129bede34694d8bb25c9b914c0e0f74 is 50, key is test_row_0/B:col10/1733595360663/Put/seqid=0 2024-12-07T18:16:02,262 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/c10cbfa0b55f4d449ff3a2d8f03b1f80 is 50, key is test_row_0/A:col10/1733595360663/Put/seqid=0 2024-12-07T18:16:02,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742363_1539 (size=13153) 2024-12-07T18:16:02,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742364_1540 (size=13153) 2024-12-07T18:16:02,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-07T18:16:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:02,298 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-07T18:16:02,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:16:02,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:02,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:16:02,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:02,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:16:02,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:16:02,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. as already flushing 2024-12-07T18:16:02,299 INFO [Thread-2078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2078. 2024-12-07T18:16:02,299 DEBUG [Thread-2078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:02,299 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:02,299 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:02,299 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:02,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/c55df8da857e4f4390ccfb8cdc549e02 is 50, key is test_row_0/A:col10/1733595361093/Put/seqid=0 2024-12-07T18:16:02,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742365_1541 (size=12301) 2024-12-07T18:16:02,669 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/9129bede34694d8bb25c9b914c0e0f74 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/9129bede34694d8bb25c9b914c0e0f74 2024-12-07T18:16:02,669 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/c10cbfa0b55f4d449ff3a2d8f03b1f80 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c10cbfa0b55f4d449ff3a2d8f03b1f80 2024-12-07T18:16:02,673 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/B of 0fcb6359e763d0a744acdb75bfd0a652 into 9129bede34694d8bb25c9b914c0e0f74(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:02,673 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/A of 0fcb6359e763d0a744acdb75bfd0a652 into c10cbfa0b55f4d449ff3a2d8f03b1f80(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:02,673 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:16:02,673 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:16:02,673 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/B, priority=13, startTime=1733595362255; duration=0sec 2024-12-07T18:16:02,673 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/A, priority=13, startTime=1733595362255; duration=0sec 2024-12-07T18:16:02,673 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:02,673 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:B 2024-12-07T18:16:02,673 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:02,673 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:A 2024-12-07T18:16:02,673 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:02,674 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:02,674 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): 0fcb6359e763d0a744acdb75bfd0a652/C is initiating minor compaction (all files) 2024-12-07T18:16:02,674 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0fcb6359e763d0a744acdb75bfd0a652/C in TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:02,674 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf4cb87245424b8ea778ab37a0bb59ee, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/909480df338149a9a8f3b4c7400fc983, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f0c7113411a5437a9efb48c8cb99a955] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp, totalSize=36.8 K 2024-12-07T18:16:02,674 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting bf4cb87245424b8ea778ab37a0bb59ee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1733595358074 2024-12-07T18:16:02,674 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 909480df338149a9a8f3b4c7400fc983, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1733595359296 2024-12-07T18:16:02,674 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting f0c7113411a5437a9efb48c8cb99a955, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733595360650 2024-12-07T18:16:02,679 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0fcb6359e763d0a744acdb75bfd0a652#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:02,680 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/686142a1a2994d5cb3197fc34cbd630d is 50, key is test_row_0/C:col10/1733595360663/Put/seqid=0 2024-12-07T18:16:02,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742366_1542 (size=13153) 2024-12-07T18:16:02,705 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/c55df8da857e4f4390ccfb8cdc549e02 2024-12-07T18:16:02,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/fa2bfd1b924247aa96ceb97fc1245d3f is 50, key is test_row_0/B:col10/1733595361093/Put/seqid=0 2024-12-07T18:16:02,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742367_1543 (size=12301) 2024-12-07T18:16:03,086 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/686142a1a2994d5cb3197fc34cbd630d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/686142a1a2994d5cb3197fc34cbd630d 2024-12-07T18:16:03,090 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0fcb6359e763d0a744acdb75bfd0a652/C of 0fcb6359e763d0a744acdb75bfd0a652 into 686142a1a2994d5cb3197fc34cbd630d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:03,090 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:16:03,090 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., storeName=0fcb6359e763d0a744acdb75bfd0a652/C, priority=13, startTime=1733595362255; duration=0sec 2024-12-07T18:16:03,090 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:03,090 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0fcb6359e763d0a744acdb75bfd0a652:C 2024-12-07T18:16:03,113 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/fa2bfd1b924247aa96ceb97fc1245d3f 2024-12-07T18:16:03,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/b892690e895841dc90d00964b73bdf49 is 50, key is test_row_0/C:col10/1733595361093/Put/seqid=0 2024-12-07T18:16:03,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742368_1544 (size=12301) 2024-12-07T18:16:03,499 INFO [Thread-2072 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2072. 2024-12-07T18:16:03,500 DEBUG [Thread-2072 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:03,500 DEBUG [Thread-2072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:03,500 DEBUG [Thread-2072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:03,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:03,520 INFO [Thread-2074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2074. 2024-12-07T18:16:03,520 DEBUG [Thread-2074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:03,520 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:03,520 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:03,520 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:03,522 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/b892690e895841dc90d00964b73bdf49 2024-12-07T18:16:03,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/c55df8da857e4f4390ccfb8cdc549e02 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c55df8da857e4f4390ccfb8cdc549e02 2024-12-07T18:16:03,528 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c55df8da857e4f4390ccfb8cdc549e02, entries=150, sequenceid=342, filesize=12.0 K 2024-12-07T18:16:03,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/fa2bfd1b924247aa96ceb97fc1245d3f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/fa2bfd1b924247aa96ceb97fc1245d3f 2024-12-07T18:16:03,529 INFO [Thread-2076 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2076. 2024-12-07T18:16:03,529 DEBUG [Thread-2076 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:03,529 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:03,529 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:03,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:03,531 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/fa2bfd1b924247aa96ceb97fc1245d3f, entries=150, sequenceid=342, filesize=12.0 K 2024-12-07T18:16:03,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/b892690e895841dc90d00964b73bdf49 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/b892690e895841dc90d00964b73bdf49 2024-12-07T18:16:03,534 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/b892690e895841dc90d00964b73bdf49, entries=150, sequenceid=342, filesize=12.0 K 2024-12-07T18:16:03,535 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for 0fcb6359e763d0a744acdb75bfd0a652 in 1237ms, sequenceid=342, compaction requested=false 2024-12-07T18:16:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 0fcb6359e763d0a744acdb75bfd0a652: 2024-12-07T18:16:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-07T18:16:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-07T18:16:03,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-07T18:16:03,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1440 sec 2024-12-07T18:16:03,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 4.1480 sec 2024-12-07T18:16:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-07T18:16:03,548 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:05,429 INFO [Thread-2070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2070. 2024-12-07T18:16:05,429 DEBUG [Thread-2070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:05,429 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:05,430 DEBUG [Thread-2070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:05,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1093 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3274 rows 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1086 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3258 rows 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1086 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3256 rows 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1090 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3265 rows 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1093 2024-12-07T18:16:05,430 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3279 rows 2024-12-07T18:16:05,430 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:16:05,430 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:16:05,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:05,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:05,430 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:05,431 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T18:16:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-07T18:16:05,434 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595365433"}]},"ts":"1733595365433"} 2024-12-07T18:16:05,435 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T18:16:05,435 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T18:16:05,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T18:16:05,435 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T18:16:05,436 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fcb6359e763d0a744acdb75bfd0a652, UNASSIGN}] 2024-12-07T18:16:05,437 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fcb6359e763d0a744acdb75bfd0a652, UNASSIGN 2024-12-07T18:16:05,437 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=0fcb6359e763d0a744acdb75bfd0a652, regionState=CLOSING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:05,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fcb6359e763d0a744acdb75bfd0a652, UNASSIGN because future has completed 2024-12-07T18:16:05,439 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T18:16:05,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:16:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-07T18:16:05,591 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 0fcb6359e763d0a744acdb75bfd0a652, disabling compactions & flushes 2024-12-07T18:16:05,591 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. after waiting 0 ms 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:05,591 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 0fcb6359e763d0a744acdb75bfd0a652 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=A 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=B 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0fcb6359e763d0a744acdb75bfd0a652, store=C 2024-12-07T18:16:05,591 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:05,594 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/8e76aba926b64387a9eca4692938c0a6 is 50, key is test_row_1/A:col10/1733595363528/Put/seqid=0 2024-12-07T18:16:05,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742369_1545 (size=9857) 2024-12-07T18:16:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-07T18:16:05,998 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/8e76aba926b64387a9eca4692938c0a6 2024-12-07T18:16:06,003 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/e73b50cf7a8e4b46877cfb009a3b9bf4 is 50, key is test_row_1/B:col10/1733595363528/Put/seqid=0 2024-12-07T18:16:06,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742370_1546 (size=9857) 2024-12-07T18:16:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-07T18:16:06,407 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/e73b50cf7a8e4b46877cfb009a3b9bf4 2024-12-07T18:16:06,412 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/9281688ca759443cab1d56063662774c is 50, key is test_row_1/C:col10/1733595363528/Put/seqid=0 2024-12-07T18:16:06,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742371_1547 (size=9857) 2024-12-07T18:16:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-07T18:16:06,816 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/9281688ca759443cab1d56063662774c 2024-12-07T18:16:06,819 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/A/8e76aba926b64387a9eca4692938c0a6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/8e76aba926b64387a9eca4692938c0a6 2024-12-07T18:16:06,822 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/8e76aba926b64387a9eca4692938c0a6, entries=100, sequenceid=353, filesize=9.6 K 2024-12-07T18:16:06,822 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/B/e73b50cf7a8e4b46877cfb009a3b9bf4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/e73b50cf7a8e4b46877cfb009a3b9bf4 2024-12-07T18:16:06,824 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/e73b50cf7a8e4b46877cfb009a3b9bf4, entries=100, sequenceid=353, filesize=9.6 K 2024-12-07T18:16:06,825 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/.tmp/C/9281688ca759443cab1d56063662774c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/9281688ca759443cab1d56063662774c 2024-12-07T18:16:06,827 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/9281688ca759443cab1d56063662774c, entries=100, sequenceid=353, filesize=9.6 K 2024-12-07T18:16:06,828 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0fcb6359e763d0a744acdb75bfd0a652 in 1237ms, sequenceid=353, compaction requested=true 2024-12-07T18:16:06,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/fd4a3465611846498bce02b85268fb91, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/145fa59fd6a5404ba3abe2cf5f222f3d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/1af6078cca164d049c50d6116a0eac94, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/90988ac2a7ec449a934e214cf244d357, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c3bd0120a41142c7a7b27569b30372e5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/01be43a3ed73470f855a4c633ff9a4b5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/23876481f9404559a4b0734c75a3a7bd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/e1bed2d45b594546bdfb2d2b9f775d5a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/b5346ec1dcd64476997d18e1c2a5f50c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/71d1ba89e9884738bf515a62a6e619e1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0bc217077c474963914f8d243268beca, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0db967f316be4e42acf3ca2a69865f43, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/28213b58eb394bf58352397ac526690d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/cf62761f28a244df93036cd12c981049, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/6c825ba98b5948d4a44f27f18461cd3c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c1ba5dfca91649c48e15d57f2067939a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/44745b3aa2c94dd19dc250762b49f15e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/08e003dd58724b50b70a909609c205ee, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/8bafc920bfe347328c05ceb4ee6f0b6b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/60f4bf4da411411b9fb2b3655f177814, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/51ab9407b7444b6cbeef75522b27af39, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0a9cfb5dbaa9468f82970db8e970a5ac, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/d3114f2bfecc47d38f46bcfb9b4ffcf2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/ba156387961e4cf2b207b0b18003568a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/aff9cb16a8c443db977d8ad4b0a90068] to archive 2024-12-07T18:16:06,829 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:16:06,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/fd4a3465611846498bce02b85268fb91 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/fd4a3465611846498bce02b85268fb91 2024-12-07T18:16:06,831 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/145fa59fd6a5404ba3abe2cf5f222f3d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/145fa59fd6a5404ba3abe2cf5f222f3d 2024-12-07T18:16:06,832 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/1af6078cca164d049c50d6116a0eac94 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/1af6078cca164d049c50d6116a0eac94 2024-12-07T18:16:06,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/90988ac2a7ec449a934e214cf244d357 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/90988ac2a7ec449a934e214cf244d357 2024-12-07T18:16:06,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c3bd0120a41142c7a7b27569b30372e5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c3bd0120a41142c7a7b27569b30372e5 2024-12-07T18:16:06,834 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/01be43a3ed73470f855a4c633ff9a4b5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/01be43a3ed73470f855a4c633ff9a4b5 2024-12-07T18:16:06,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/23876481f9404559a4b0734c75a3a7bd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/23876481f9404559a4b0734c75a3a7bd 2024-12-07T18:16:06,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/e1bed2d45b594546bdfb2d2b9f775d5a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/e1bed2d45b594546bdfb2d2b9f775d5a 2024-12-07T18:16:06,836 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/b5346ec1dcd64476997d18e1c2a5f50c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/b5346ec1dcd64476997d18e1c2a5f50c 2024-12-07T18:16:06,837 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/71d1ba89e9884738bf515a62a6e619e1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/71d1ba89e9884738bf515a62a6e619e1 2024-12-07T18:16:06,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0bc217077c474963914f8d243268beca to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0bc217077c474963914f8d243268beca 2024-12-07T18:16:06,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0db967f316be4e42acf3ca2a69865f43 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0db967f316be4e42acf3ca2a69865f43 2024-12-07T18:16:06,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/28213b58eb394bf58352397ac526690d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/28213b58eb394bf58352397ac526690d 2024-12-07T18:16:06,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/cf62761f28a244df93036cd12c981049 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/cf62761f28a244df93036cd12c981049 2024-12-07T18:16:06,840 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/6c825ba98b5948d4a44f27f18461cd3c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/6c825ba98b5948d4a44f27f18461cd3c 2024-12-07T18:16:06,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c1ba5dfca91649c48e15d57f2067939a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c1ba5dfca91649c48e15d57f2067939a 2024-12-07T18:16:06,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/44745b3aa2c94dd19dc250762b49f15e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/44745b3aa2c94dd19dc250762b49f15e 2024-12-07T18:16:06,842 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/08e003dd58724b50b70a909609c205ee to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/08e003dd58724b50b70a909609c205ee 2024-12-07T18:16:06,843 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/8bafc920bfe347328c05ceb4ee6f0b6b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/8bafc920bfe347328c05ceb4ee6f0b6b 2024-12-07T18:16:06,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/60f4bf4da411411b9fb2b3655f177814 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/60f4bf4da411411b9fb2b3655f177814 2024-12-07T18:16:06,844 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/51ab9407b7444b6cbeef75522b27af39 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/51ab9407b7444b6cbeef75522b27af39 2024-12-07T18:16:06,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0a9cfb5dbaa9468f82970db8e970a5ac to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/0a9cfb5dbaa9468f82970db8e970a5ac 2024-12-07T18:16:06,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/d3114f2bfecc47d38f46bcfb9b4ffcf2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/d3114f2bfecc47d38f46bcfb9b4ffcf2 2024-12-07T18:16:06,846 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/ba156387961e4cf2b207b0b18003568a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/ba156387961e4cf2b207b0b18003568a 2024-12-07T18:16:06,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/aff9cb16a8c443db977d8ad4b0a90068 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/aff9cb16a8c443db977d8ad4b0a90068 2024-12-07T18:16:06,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c552217def7f4d59865798ed863aa2ae, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/bd8d7ea66bbb48169f60e7a637fbbbe5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/855d1cc268c7476f8f5d8682dbdc1b59, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/eeb3bcf4b60e4a6bac6a81e26ec8e321, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/3a3eb905e69740aaa078f80772c12fa0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/cbb24907e0d14847be568483648b7f49, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/232dcc1e14d14026a2c53c8310dd5f78, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/1b06645affdf48cc8ee7a083e3495762, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c645ed55f6b84d029688cbfd145f0f5e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/b7c0d87bc67a438e999da08fee80a193, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/991297f248ac48cfb4da92c558ef8dfe, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c8def2fe5d2948b7a2c3c9d3e4bd07e1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/217e8202073f4fe2b572cd43c5b81967, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/fd9e3ee92f344a5a9763a697e300377f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7df8267fae9548fe817f44e58f4c6cbb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/8248911b25034e4093e3eee62a32b9dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/adef8eccc20a4273843b3c98d98d91cb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/41a5af55d81f44caab3a36801da9e5b3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/d56b70920aac4574885c27eb03c7904e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/caebecd378e64a86b62745101d801153, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/a7082d7a1f85420bb31256d4b0025acc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/49f364b75b774d45abd6d334f617652f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/99fad172cc2d424b9cec18b27bc3376c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7daa09915ba746b49dd35557033c71f8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/ba61e628efe543809b13f332ad6b855f] to archive 2024-12-07T18:16:06,849 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:16:06,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c552217def7f4d59865798ed863aa2ae to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c552217def7f4d59865798ed863aa2ae 2024-12-07T18:16:06,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/bd8d7ea66bbb48169f60e7a637fbbbe5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/bd8d7ea66bbb48169f60e7a637fbbbe5 2024-12-07T18:16:06,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/855d1cc268c7476f8f5d8682dbdc1b59 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/855d1cc268c7476f8f5d8682dbdc1b59 2024-12-07T18:16:06,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/eeb3bcf4b60e4a6bac6a81e26ec8e321 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/eeb3bcf4b60e4a6bac6a81e26ec8e321 2024-12-07T18:16:06,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/3a3eb905e69740aaa078f80772c12fa0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/3a3eb905e69740aaa078f80772c12fa0 2024-12-07T18:16:06,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/cbb24907e0d14847be568483648b7f49 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/cbb24907e0d14847be568483648b7f49 2024-12-07T18:16:06,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/232dcc1e14d14026a2c53c8310dd5f78 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/232dcc1e14d14026a2c53c8310dd5f78 2024-12-07T18:16:06,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/1b06645affdf48cc8ee7a083e3495762 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/1b06645affdf48cc8ee7a083e3495762 2024-12-07T18:16:06,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c645ed55f6b84d029688cbfd145f0f5e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c645ed55f6b84d029688cbfd145f0f5e 2024-12-07T18:16:06,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/b7c0d87bc67a438e999da08fee80a193 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/b7c0d87bc67a438e999da08fee80a193 2024-12-07T18:16:06,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/991297f248ac48cfb4da92c558ef8dfe to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/991297f248ac48cfb4da92c558ef8dfe 2024-12-07T18:16:06,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c8def2fe5d2948b7a2c3c9d3e4bd07e1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/c8def2fe5d2948b7a2c3c9d3e4bd07e1 2024-12-07T18:16:06,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/217e8202073f4fe2b572cd43c5b81967 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/217e8202073f4fe2b572cd43c5b81967 2024-12-07T18:16:06,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/fd9e3ee92f344a5a9763a697e300377f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/fd9e3ee92f344a5a9763a697e300377f 2024-12-07T18:16:06,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7df8267fae9548fe817f44e58f4c6cbb to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7df8267fae9548fe817f44e58f4c6cbb 2024-12-07T18:16:06,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/8248911b25034e4093e3eee62a32b9dd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/8248911b25034e4093e3eee62a32b9dd 2024-12-07T18:16:06,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/adef8eccc20a4273843b3c98d98d91cb to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/adef8eccc20a4273843b3c98d98d91cb 2024-12-07T18:16:06,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/41a5af55d81f44caab3a36801da9e5b3 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/41a5af55d81f44caab3a36801da9e5b3 2024-12-07T18:16:06,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/d56b70920aac4574885c27eb03c7904e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/d56b70920aac4574885c27eb03c7904e 2024-12-07T18:16:06,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/caebecd378e64a86b62745101d801153 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/caebecd378e64a86b62745101d801153 2024-12-07T18:16:06,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/a7082d7a1f85420bb31256d4b0025acc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/a7082d7a1f85420bb31256d4b0025acc 2024-12-07T18:16:06,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/49f364b75b774d45abd6d334f617652f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/49f364b75b774d45abd6d334f617652f 2024-12-07T18:16:06,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/99fad172cc2d424b9cec18b27bc3376c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/99fad172cc2d424b9cec18b27bc3376c 2024-12-07T18:16:06,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7daa09915ba746b49dd35557033c71f8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/7daa09915ba746b49dd35557033c71f8 2024-12-07T18:16:06,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/ba61e628efe543809b13f332ad6b855f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/ba61e628efe543809b13f332ad6b855f 2024-12-07T18:16:06,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/3a7d80046bfa48f9bf62b8982fae8c24, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ff3921a3f3a048e79fb83564754d65bd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/901c7e114572491c9b3206525d6e1006, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebff422f90bc440ea465f84de8877115, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0c9060837caf4d4983aee6efabfd43a7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/2bdd8d37287b44f380978889fedc907b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a08c50f2568847c09e52e73f7abd836e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/9c79d4f5dee74e23b38fe971e3b1861f, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/90672bd18309430da81ad5f7be066cef, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/17935b92e8194d93b7959b6c21d04fe3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/4d2dd16cf1da42f7819cb8c5db994cc7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bbc147d50a224e6bbc7a0a0a7b0a23a4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/7249527c8c834908b1b3432141f790dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a006e5ec3d574c179aaa818a1e5d128a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f7d8dc2094ba4ad7b843c68f1be8493d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf5e8a1e1a0843f487131de9f35a9f8b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf83d3cae982472da3429aec96e773f4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebfc77d500a74634b7f53e4c01c0aa8e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/2f0d938586ba41958b15716555c1b223, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bdcb21daba46435e85b58bdc9873ab0b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0d11818bffe74845858823f0daf70427, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf4cb87245424b8ea778ab37a0bb59ee, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/d7cf0ecc53ad4e32a9db956a7f027335, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/909480df338149a9a8f3b4c7400fc983, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f0c7113411a5437a9efb48c8cb99a955] to archive 2024-12-07T18:16:06,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:16:06,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/3a7d80046bfa48f9bf62b8982fae8c24 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/3a7d80046bfa48f9bf62b8982fae8c24 2024-12-07T18:16:06,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ff3921a3f3a048e79fb83564754d65bd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ff3921a3f3a048e79fb83564754d65bd 2024-12-07T18:16:06,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/901c7e114572491c9b3206525d6e1006 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/901c7e114572491c9b3206525d6e1006 2024-12-07T18:16:06,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebff422f90bc440ea465f84de8877115 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebff422f90bc440ea465f84de8877115 2024-12-07T18:16:06,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0c9060837caf4d4983aee6efabfd43a7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0c9060837caf4d4983aee6efabfd43a7 2024-12-07T18:16:06,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/2bdd8d37287b44f380978889fedc907b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/2bdd8d37287b44f380978889fedc907b 2024-12-07T18:16:06,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a08c50f2568847c09e52e73f7abd836e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a08c50f2568847c09e52e73f7abd836e 2024-12-07T18:16:06,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/9c79d4f5dee74e23b38fe971e3b1861f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/9c79d4f5dee74e23b38fe971e3b1861f 2024-12-07T18:16:06,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/90672bd18309430da81ad5f7be066cef to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/90672bd18309430da81ad5f7be066cef 2024-12-07T18:16:06,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/17935b92e8194d93b7959b6c21d04fe3 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/17935b92e8194d93b7959b6c21d04fe3 2024-12-07T18:16:06,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/4d2dd16cf1da42f7819cb8c5db994cc7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/4d2dd16cf1da42f7819cb8c5db994cc7 2024-12-07T18:16:06,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bbc147d50a224e6bbc7a0a0a7b0a23a4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bbc147d50a224e6bbc7a0a0a7b0a23a4 2024-12-07T18:16:06,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/7249527c8c834908b1b3432141f790dd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/7249527c8c834908b1b3432141f790dd 2024-12-07T18:16:06,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a006e5ec3d574c179aaa818a1e5d128a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/a006e5ec3d574c179aaa818a1e5d128a 2024-12-07T18:16:06,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f7d8dc2094ba4ad7b843c68f1be8493d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f7d8dc2094ba4ad7b843c68f1be8493d 2024-12-07T18:16:06,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf5e8a1e1a0843f487131de9f35a9f8b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf5e8a1e1a0843f487131de9f35a9f8b 2024-12-07T18:16:06,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf83d3cae982472da3429aec96e773f4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf83d3cae982472da3429aec96e773f4 2024-12-07T18:16:06,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebfc77d500a74634b7f53e4c01c0aa8e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/ebfc77d500a74634b7f53e4c01c0aa8e 2024-12-07T18:16:06,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/2f0d938586ba41958b15716555c1b223 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/2f0d938586ba41958b15716555c1b223 2024-12-07T18:16:06,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bdcb21daba46435e85b58bdc9873ab0b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bdcb21daba46435e85b58bdc9873ab0b 2024-12-07T18:16:06,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0d11818bffe74845858823f0daf70427 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/0d11818bffe74845858823f0daf70427 2024-12-07T18:16:06,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf4cb87245424b8ea778ab37a0bb59ee to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/bf4cb87245424b8ea778ab37a0bb59ee 2024-12-07T18:16:06,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/d7cf0ecc53ad4e32a9db956a7f027335 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/d7cf0ecc53ad4e32a9db956a7f027335 2024-12-07T18:16:06,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/909480df338149a9a8f3b4c7400fc983 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/909480df338149a9a8f3b4c7400fc983 2024-12-07T18:16:06,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f0c7113411a5437a9efb48c8cb99a955 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/f0c7113411a5437a9efb48c8cb99a955 2024-12-07T18:16:06,893 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/recovered.edits/356.seqid, newMaxSeqId=356, maxSeqId=1 2024-12-07T18:16:06,893 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. 2024-12-07T18:16:06,893 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 0fcb6359e763d0a744acdb75bfd0a652: Waiting for close lock at 1733595365591Running coprocessor pre-close hooks at 1733595365591Disabling compacts and flushes for region at 1733595365591Disabling writes for close at 1733595365591Obtaining lock to block concurrent updates at 1733595365591Preparing flush snapshotting stores in 0fcb6359e763d0a744acdb75bfd0a652 at 1733595365591Finished memstore snapshotting TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733595365591Flushing stores of TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652. at 1733595365592 (+1 ms)Flushing 0fcb6359e763d0a744acdb75bfd0a652/A: creating writer at 1733595365592Flushing 0fcb6359e763d0a744acdb75bfd0a652/A: appending metadata at 1733595365594 (+2 ms)Flushing 0fcb6359e763d0a744acdb75bfd0a652/A: closing flushed file at 1733595365594Flushing 0fcb6359e763d0a744acdb75bfd0a652/B: creating writer at 1733595366001 (+407 ms)Flushing 0fcb6359e763d0a744acdb75bfd0a652/B: appending metadata at 1733595366003 (+2 ms)Flushing 0fcb6359e763d0a744acdb75bfd0a652/B: closing flushed file at 1733595366003Flushing 0fcb6359e763d0a744acdb75bfd0a652/C: creating writer at 1733595366410 (+407 ms)Flushing 0fcb6359e763d0a744acdb75bfd0a652/C: appending metadata at 1733595366412 (+2 ms)Flushing 0fcb6359e763d0a744acdb75bfd0a652/C: closing flushed file at 1733595366412Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67fc407f: reopening flushed file at 1733595366818 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40783f59: reopening flushed file at 1733595366822 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14e3df5c: reopening flushed file at 1733595366825 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0fcb6359e763d0a744acdb75bfd0a652 in 1237ms, sequenceid=353, compaction requested=true at 1733595366828 (+3 ms)Writing region close event to WAL at 1733595366890 (+62 ms)Running coprocessor post-close hooks at 1733595366893 (+3 ms)Closed at 1733595366893 2024-12-07T18:16:06,895 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:16:06,896 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=0fcb6359e763d0a744acdb75bfd0a652, regionState=CLOSED 2024-12-07T18:16:06,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:16:06,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-07T18:16:06,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 0fcb6359e763d0a744acdb75bfd0a652, server=92acd7e2389b,33947,1733595217075 in 1.4580 sec 2024-12-07T18:16:06,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-07T18:16:06,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0fcb6359e763d0a744acdb75bfd0a652, UNASSIGN in 1.4630 sec 2024-12-07T18:16:06,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-07T18:16:06,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4650 sec 2024-12-07T18:16:06,902 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595366902"}]},"ts":"1733595366902"} 2024-12-07T18:16:06,903 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T18:16:06,903 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T18:16:06,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4730 sec 2024-12-07T18:16:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-07T18:16:07,579 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:07,579 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T18:16:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:07,581 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-07T18:16:07,582 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:07,583 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:16:07,585 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/recovered.edits] 2024-12-07T18:16:07,587 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/8e76aba926b64387a9eca4692938c0a6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/8e76aba926b64387a9eca4692938c0a6 2024-12-07T18:16:07,588 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c10cbfa0b55f4d449ff3a2d8f03b1f80 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c10cbfa0b55f4d449ff3a2d8f03b1f80 2024-12-07T18:16:07,589 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c55df8da857e4f4390ccfb8cdc549e02 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/A/c55df8da857e4f4390ccfb8cdc549e02 2024-12-07T18:16:07,590 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/9129bede34694d8bb25c9b914c0e0f74 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/9129bede34694d8bb25c9b914c0e0f74 2024-12-07T18:16:07,591 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/e73b50cf7a8e4b46877cfb009a3b9bf4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/e73b50cf7a8e4b46877cfb009a3b9bf4 2024-12-07T18:16:07,592 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/fa2bfd1b924247aa96ceb97fc1245d3f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/B/fa2bfd1b924247aa96ceb97fc1245d3f 2024-12-07T18:16:07,594 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/686142a1a2994d5cb3197fc34cbd630d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/686142a1a2994d5cb3197fc34cbd630d 2024-12-07T18:16:07,595 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/9281688ca759443cab1d56063662774c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/9281688ca759443cab1d56063662774c 2024-12-07T18:16:07,596 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/b892690e895841dc90d00964b73bdf49 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/C/b892690e895841dc90d00964b73bdf49 2024-12-07T18:16:07,598 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/recovered.edits/356.seqid to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652/recovered.edits/356.seqid 2024-12-07T18:16:07,599 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/0fcb6359e763d0a744acdb75bfd0a652 2024-12-07T18:16:07,599 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T18:16:07,601 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:07,602 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T18:16:07,604 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T18:16:07,605 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:07,605 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T18:16:07,605 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733595367605"}]},"ts":"9223372036854775807"} 2024-12-07T18:16:07,607 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T18:16:07,607 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0fcb6359e763d0a744acdb75bfd0a652, NAME => 'TestAcidGuarantees,,1733595339137.0fcb6359e763d0a744acdb75bfd0a652.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T18:16:07,607 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T18:16:07,607 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733595367607"}]},"ts":"9223372036854775807"} 2024-12-07T18:16:07,608 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T18:16:07,609 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:07,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-12-07T18:16:07,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-07T18:16:07,689 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T18:16:07,689 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:07,698 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 252), OpenFileDescriptor=447 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 430), ProcessCount=11 (was 11), AvailableMemoryMB=5849 (was 5922) 2024-12-07T18:16:07,706 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=5849 2024-12-07T18:16:07,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T18:16:07,707 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:16:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:07,709 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T18:16:07,709 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:07,709 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-12-07T18:16:07,710 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T18:16:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-07T18:16:07,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742372_1548 (size=1061) 2024-12-07T18:16:07,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-07T18:16:08,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-07T18:16:08,116 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68 2024-12-07T18:16:08,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742373_1549 (size=53) 2024-12-07T18:16:08,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-07T18:16:08,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:16:08,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ec33782b2c22c62c1e44260bfbcf6877, disabling compactions & flushes 2024-12-07T18:16:08,521 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:08,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:08,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. after waiting 0 ms 2024-12-07T18:16:08,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:08,521 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:08,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ec33782b2c22c62c1e44260bfbcf6877: Waiting for close lock at 1733595368521Disabling compacts and flushes for region at 1733595368521Disabling writes for close at 1733595368521Writing region close event to WAL at 1733595368521Closed at 1733595368521 2024-12-07T18:16:08,522 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T18:16:08,522 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733595368522"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733595368522"}]},"ts":"1733595368522"} 2024-12-07T18:16:08,523 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T18:16:08,524 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T18:16:08,524 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595368524"}]},"ts":"1733595368524"} 2024-12-07T18:16:08,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-07T18:16:08,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, ASSIGN}] 2024-12-07T18:16:08,526 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, ASSIGN 2024-12-07T18:16:08,527 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, ASSIGN; state=OFFLINE, location=92acd7e2389b,33947,1733595217075; forceNewPlan=false, retain=false 2024-12-07T18:16:08,677 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=ec33782b2c22c62c1e44260bfbcf6877, regionState=OPENING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:08,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, ASSIGN because future has completed 2024-12-07T18:16:08,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:16:08,833 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:08,833 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:16:08,833 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,833 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:16:08,833 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,833 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,834 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,836 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:16:08,836 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec33782b2c22c62c1e44260bfbcf6877 columnFamilyName A 2024-12-07T18:16:08,836 DEBUG [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:08,836 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(327): Store=ec33782b2c22c62c1e44260bfbcf6877/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:16:08,836 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,837 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:16:08,837 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec33782b2c22c62c1e44260bfbcf6877 columnFamilyName B 2024-12-07T18:16:08,837 DEBUG [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:08,838 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(327): Store=ec33782b2c22c62c1e44260bfbcf6877/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:16:08,838 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,838 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:16:08,838 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec33782b2c22c62c1e44260bfbcf6877 columnFamilyName C 2024-12-07T18:16:08,838 DEBUG [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:08,839 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(327): Store=ec33782b2c22c62c1e44260bfbcf6877/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:16:08,839 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:08,839 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,839 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,840 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,840 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,840 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,841 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:16:08,842 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,843 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T18:16:08,843 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened ec33782b2c22c62c1e44260bfbcf6877; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74978055, jitterRate=0.11726008355617523}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:16:08,843 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:08,844 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for ec33782b2c22c62c1e44260bfbcf6877: Running coprocessor pre-open hook at 1733595368833Writing region info on filesystem at 1733595368833Initializing all the Stores at 1733595368834 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595368834Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595368834Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595368834Cleaning up temporary data from old regions at 1733595368840 (+6 ms)Running coprocessor post-open hooks at 1733595368843 (+3 ms)Region opened successfully at 1733595368844 (+1 ms) 2024-12-07T18:16:08,844 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., pid=146, masterSystemTime=1733595368830 2024-12-07T18:16:08,846 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:08,846 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:08,847 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=ec33782b2c22c62c1e44260bfbcf6877, regionState=OPEN, openSeqNum=2, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:08,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:16:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-07T18:16:08,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-07T18:16:08,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 in 169 msec 2024-12-07T18:16:08,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-07T18:16:08,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, ASSIGN in 325 msec 2024-12-07T18:16:08,852 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T18:16:08,852 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595368852"}]},"ts":"1733595368852"} 2024-12-07T18:16:08,853 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-07T18:16:08,854 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T18:16:08,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-12-07T18:16:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-07T18:16:09,858 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:09,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1832522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:09,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:09,860 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:09,860 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbec9c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:09,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:09,862 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:09,862 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1567b1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:09,863 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:09,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:09,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:09,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:09,865 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T18:16:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 92acd7e2389b,38531,1733595216342 2024-12-07T18:16:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ebbeefb 2024-12-07T18:16:09,866 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T18:16:09,867 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T18:16:09,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-07T18:16:09,868 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T18:16:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:09,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742374_1550 (size=1097) 2024-12-07T18:16:10,279 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-07T18:16:10,279 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-07T18:16:10,281 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T18:16:10,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, REOPEN/MOVE}] 2024-12-07T18:16:10,283 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, REOPEN/MOVE 2024-12-07T18:16:10,284 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ec33782b2c22c62c1e44260bfbcf6877, regionState=CLOSING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,285 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, REOPEN/MOVE because future has completed 2024-12-07T18:16:10,285 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T18:16:10,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:16:10,437 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,437 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T18:16:10,437 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing ec33782b2c22c62c1e44260bfbcf6877, disabling compactions & flushes 2024-12-07T18:16:10,437 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:10,437 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:10,437 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. after waiting 0 ms 2024-12-07T18:16:10,437 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:10,441 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-07T18:16:10,442 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:10,442 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for ec33782b2c22c62c1e44260bfbcf6877: Waiting for close lock at 1733595370437Running coprocessor pre-close hooks at 1733595370437Disabling compacts and flushes for region at 1733595370437Disabling writes for close at 1733595370437Writing region close event to WAL at 1733595370438 (+1 ms)Running coprocessor post-close hooks at 1733595370442 (+4 ms)Closed at 1733595370442 2024-12-07T18:16:10,442 WARN [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: ec33782b2c22c62c1e44260bfbcf6877 to self. 2024-12-07T18:16:10,444 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,444 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ec33782b2c22c62c1e44260bfbcf6877, regionState=CLOSED 2024-12-07T18:16:10,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:16:10,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-07T18:16:10,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 in 161 msec 2024-12-07T18:16:10,448 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, REOPEN/MOVE; state=CLOSED, location=92acd7e2389b,33947,1733595217075; forceNewPlan=false, retain=true 2024-12-07T18:16:10,599 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ec33782b2c22c62c1e44260bfbcf6877, regionState=OPENING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, REOPEN/MOVE because future has completed 2024-12-07T18:16:10,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:16:10,754 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:10,754 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} 2024-12-07T18:16:10,755 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,755 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T18:16:10,755 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,755 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,756 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,757 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:16:10,757 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec33782b2c22c62c1e44260bfbcf6877 columnFamilyName A 2024-12-07T18:16:10,758 DEBUG [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:10,758 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(327): Store=ec33782b2c22c62c1e44260bfbcf6877/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:16:10,758 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,759 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:16:10,759 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec33782b2c22c62c1e44260bfbcf6877 columnFamilyName B 2024-12-07T18:16:10,759 DEBUG [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:10,759 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(327): Store=ec33782b2c22c62c1e44260bfbcf6877/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:16:10,759 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,760 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-07T18:16:10,760 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec33782b2c22c62c1e44260bfbcf6877 columnFamilyName C 2024-12-07T18:16:10,760 DEBUG [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:10,760 INFO [StoreOpener-ec33782b2c22c62c1e44260bfbcf6877-1 {}] regionserver.HStore(327): Store=ec33782b2c22c62c1e44260bfbcf6877/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T18:16:10,760 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:10,760 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,761 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,762 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,762 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,762 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,763 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-07T18:16:10,763 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,764 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened ec33782b2c22c62c1e44260bfbcf6877; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71818251, jitterRate=0.07017533481121063}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-07T18:16:10,764 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,765 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for ec33782b2c22c62c1e44260bfbcf6877: Running coprocessor pre-open hook at 1733595370755Writing region info on filesystem at 1733595370755Initializing all the Stores at 1733595370756 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595370756Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595370756Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733595370756Cleaning up temporary data from old regions at 1733595370762 (+6 ms)Running coprocessor post-open hooks at 1733595370764 (+2 ms)Region opened successfully at 1733595370764 2024-12-07T18:16:10,765 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., pid=151, masterSystemTime=1733595370752 2024-12-07T18:16:10,766 DEBUG [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:10,767 INFO [RS_OPEN_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:10,767 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=ec33782b2c22c62c1e44260bfbcf6877, regionState=OPEN, openSeqNum=5, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:16:10,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-12-07T18:16:10,770 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 in 169 msec 2024-12-07T18:16:10,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-07T18:16:10,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, REOPEN/MOVE in 487 msec 2024-12-07T18:16:10,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-07T18:16:10,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-12-07T18:16:10,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 904 msec 2024-12-07T18:16:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-07T18:16:10,775 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:10,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22faf5ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:10,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:10,777 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333cf25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:10,778 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:10,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ad49fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744b0f71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:10,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:10,780 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@96afea9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:10,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:10,781 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:10,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d4d24e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:10,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:10,782 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:10,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:10,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f39da38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:10,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:10,783 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:10,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@174447a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:10,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:10,785 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6389b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:10,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:10,786 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:10,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c67702e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:10,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:10,787 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:10,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2944c9b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:10,788 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5717ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,789 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:10,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:10,790 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ae0e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:10,791 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ccaaa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:10,792 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:10,792 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:10,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:10,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:10,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4babc00b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:10,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:10,793 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:10,793 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:10,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3359619a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:10,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:10,795 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65094e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:10,796 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:10,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35987f34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:10,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:10,797 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:10,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca5b581, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:10,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:10,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:10,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e59059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 92acd7e2389b,38531,-1 for getting cluster id 2024-12-07T18:16:10,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T18:16:10,800 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '035791d7-7975-48fc-9d8e-4ea192b86b0e' 2024-12-07T18:16:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T18:16:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "035791d7-7975-48fc-9d8e-4ea192b86b0e" 2024-12-07T18:16:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3a27e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [92acd7e2389b,38531,-1] 2024-12-07T18:16:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T18:16:10,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:10,801 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T18:16:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b669db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T18:16:10,802 DEBUG [Thread-2465 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:10,802 DEBUG [Thread-2448 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:10,802 DEBUG [Thread-2463 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:10,802 DEBUG [Thread-2452 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:10,802 DEBUG [Thread-2446 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:10,802 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:16:10,802 DEBUG [Thread-2457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:10,803 DEBUG [Thread-2454 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:10,803 DEBUG [Thread-2461 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:10,803 DEBUG [Thread-2450 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-07T18:16:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:10,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:10,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:10,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46782, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:10,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:10,804 DEBUG [Thread-2459 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T18:16:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:10,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:10,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:10,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=92acd7e2389b,33947,1733595217075, seqNum=-1] 2024-12-07T18:16:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T18:16:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-07T18:16:10,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:10,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:16:10,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:16:10,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:10,807 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:16:10,808 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:16:10,808 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T18:16:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:16:10,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:16:10,809 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:16:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:16:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:16:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5] 2024-12-07T18:16:10,811 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:16:10,811 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:16:10,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:16:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:10,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-07T18:16:10,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:10,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:10,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:10,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:10,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:10,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:10,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595370830, max timestamp=1733595370840Num uniques 859; 2024-12-07T18:16:10,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595370830, max timestamp=1733595370840Num uniques 859; 2024-12-07T18:16:10,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733595370830, max timestamp=1733595370839Num uniques 859; 2024-12-07T18:16:10,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:16:10,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:16:10,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:16:10,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120711fa6b37a99749fdab2b35578f1ce04c_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595370814/Put/seqid=0 2024-12-07T18:16:10,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742375_1551 (size=12154) 2024-12-07T18:16:10,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46802 deadline: 1733595430858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:10,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595430859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595430859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595430860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46788 deadline: 1733595430861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-07T18:16:10,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:10,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:10,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:10,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:10,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:10,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:10,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46802 deadline: 1733595430968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:10,970 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:10,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595430968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:10,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595430968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595430968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:10,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46788 deadline: 1733595430968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:11,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:11,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-07T18:16:11,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:11,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46802 deadline: 1733595431178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:11,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:11,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595431178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595431178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595431179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46788 deadline: 1733595431179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,251 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:11,255 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120711fa6b37a99749fdab2b35578f1ce04c_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120711fa6b37a99749fdab2b35578f1ce04c_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:11,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/b431724f3e0f4b4fb39a2537de36c0f2, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:11,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/b431724f3e0f4b4fb39a2537de36c0f2 is 175, key is test_row_0/A:col10/1733595370814/Put/seqid=0 2024-12-07T18:16:11,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742376_1552 (size=30955) 2024-12-07T18:16:11,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:11,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:11,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-07T18:16:11,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46802 deadline: 1733595431488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595431488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595431488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:11,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595431488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46788 deadline: 1733595431489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:11,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:11,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:11,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,660 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/b431724f3e0f4b4fb39a2537de36c0f2 2024-12-07T18:16:11,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/54c683fdd3f546aa84fd9f8c249bae52 is 50, key is test_row_0/B:col10/1733595370814/Put/seqid=0 2024-12-07T18:16:11,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742377_1553 (size=12001) 2024-12-07T18:16:11,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:11,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:11,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:11,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:11,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:11,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-07T18:16:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595431999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:12,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595431999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46802 deadline: 1733595432008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595432008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:12,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46788 deadline: 1733595432008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:12,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:12,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:12,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:12,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:12,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/54c683fdd3f546aa84fd9f8c249bae52 2024-12-07T18:16:12,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/94e04c56f9b749f39850420fe8399fcc is 50, key is test_row_0/C:col10/1733595370814/Put/seqid=0 2024-12-07T18:16:12,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742378_1554 (size=12001) 2024-12-07T18:16:12,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:12,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:12,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:12,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:12,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:12,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:12,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:12,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:12,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,413 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T18:16:12,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:12,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:12,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:12,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:12,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:12,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/94e04c56f9b749f39850420fe8399fcc 2024-12-07T18:16:12,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/b431724f3e0f4b4fb39a2537de36c0f2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/b431724f3e0f4b4fb39a2537de36c0f2 2024-12-07T18:16:12,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/b431724f3e0f4b4fb39a2537de36c0f2, entries=150, sequenceid=15, filesize=30.2 K 2024-12-07T18:16:12,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/54c683fdd3f546aa84fd9f8c249bae52 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/54c683fdd3f546aa84fd9f8c249bae52 2024-12-07T18:16:12,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/54c683fdd3f546aa84fd9f8c249bae52, entries=150, sequenceid=15, filesize=11.7 K 2024-12-07T18:16:12,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/94e04c56f9b749f39850420fe8399fcc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/94e04c56f9b749f39850420fe8399fcc 2024-12-07T18:16:12,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/94e04c56f9b749f39850420fe8399fcc, entries=150, sequenceid=15, filesize=11.7 K 2024-12-07T18:16:12,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ec33782b2c22c62c1e44260bfbcf6877 in 1711ms, sequenceid=15, compaction requested=false 2024-12-07T18:16:12,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:12,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-07T18:16:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:12,645 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-07T18:16:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:12,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207563a9695f9c940e48eba945a13f975be_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595370859/Put/seqid=0 2024-12-07T18:16:12,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742379_1555 (size=12154) 2024-12-07T18:16:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-07T18:16:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:13,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:13,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46802 deadline: 1733595433028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46788 deadline: 1733595433028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595433028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,033 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in ec33782b2c22c62c1e44260bfbcf6877 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:13,034 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in ec33782b2c22c62c1e44260bfbcf6877 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595433030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595433030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:13,063 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207563a9695f9c940e48eba945a13f975be_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207563a9695f9c940e48eba945a13f975be_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:13,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/2d94bf856b6b422093079756b4d4634e, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:13,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/2d94bf856b6b422093079756b4d4634e is 175, key is test_row_0/A:col10/1733595370859/Put/seqid=0 2024-12-07T18:16:13,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742380_1556 (size=30955) 2024-12-07T18:16:13,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595433138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595433138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595433138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595433348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595433348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595433348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,468 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/2d94bf856b6b422093079756b4d4634e 2024-12-07T18:16:13,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/47b0bb6dd7524cffb336decf1e4a48b3 is 50, key is test_row_0/B:col10/1733595370859/Put/seqid=0 2024-12-07T18:16:13,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742381_1557 (size=12001) 2024-12-07T18:16:13,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595433658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595433658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595433658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:13,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:13,879 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/47b0bb6dd7524cffb336decf1e4a48b3 2024-12-07T18:16:13,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/c56f306406ca4fcf952004a99076ef3c is 50, key is test_row_0/C:col10/1733595370859/Put/seqid=0 2024-12-07T18:16:13,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742382_1558 (size=12001) 2024-12-07T18:16:13,897 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/c56f306406ca4fcf952004a99076ef3c 2024-12-07T18:16:13,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/2d94bf856b6b422093079756b4d4634e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/2d94bf856b6b422093079756b4d4634e 2024-12-07T18:16:13,904 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/2d94bf856b6b422093079756b4d4634e, entries=150, sequenceid=50, filesize=30.2 K 2024-12-07T18:16:13,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/47b0bb6dd7524cffb336decf1e4a48b3 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/47b0bb6dd7524cffb336decf1e4a48b3 2024-12-07T18:16:13,908 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/47b0bb6dd7524cffb336decf1e4a48b3, entries=150, sequenceid=50, filesize=11.7 K 2024-12-07T18:16:13,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/c56f306406ca4fcf952004a99076ef3c as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/c56f306406ca4fcf952004a99076ef3c 2024-12-07T18:16:13,913 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/c56f306406ca4fcf952004a99076ef3c, entries=150, sequenceid=50, filesize=11.7 K 2024-12-07T18:16:13,914 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for ec33782b2c22c62c1e44260bfbcf6877 in 1269ms, sequenceid=50, compaction requested=false 2024-12-07T18:16:13,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:13,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:13,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-07T18:16:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-07T18:16:13,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-07T18:16:13,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1050 sec 2024-12-07T18:16:13,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 3.1170 sec 2024-12-07T18:16:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:14,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T18:16:14,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:14,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:14,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:14,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:14,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:14,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:14,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207ac6f5a4d28284c88bb23360026c6b72c_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595374169/Put/seqid=0 2024-12-07T18:16:14,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742383_1559 (size=14594) 2024-12-07T18:16:14,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595434192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595434196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595434196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595434308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595434308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595434308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595434518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595434518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595434518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,582 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:14,586 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207ac6f5a4d28284c88bb23360026c6b72c_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ac6f5a4d28284c88bb23360026c6b72c_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:14,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/82bc91843a5c4467afab88a308016b9a, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:14,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/82bc91843a5c4467afab88a308016b9a is 175, key is test_row_0/A:col10/1733595374169/Put/seqid=0 2024-12-07T18:16:14,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742384_1560 (size=39549) 2024-12-07T18:16:14,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595434828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595434829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595434829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-07T18:16:14,978 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:14,979 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:16:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-07T18:16:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-07T18:16:14,980 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:16:14,981 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:16:14,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:16:14,991 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/82bc91843a5c4467afab88a308016b9a 2024-12-07T18:16:14,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/1182147fc71e499191ff214fa8d06c99 is 50, key is test_row_0/B:col10/1733595374169/Put/seqid=0 2024-12-07T18:16:15,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742385_1561 (size=12001) 2024-12-07T18:16:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46788 deadline: 1733595435049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46802 deadline: 1733595435049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:15,053 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in ec33782b2c22c62c1e44260bfbcf6877 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:15,053 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in ec33782b2c22c62c1e44260bfbcf6877 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:15,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-07T18:16:15,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-07T18:16:15,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:15,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:15,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:15,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:15,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:15,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-07T18:16:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:15,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:15,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-07T18:16:15,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595435338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:15,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595435338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:15,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595435348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:15,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/1182147fc71e499191ff214fa8d06c99 2024-12-07T18:16:15,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/953499dccd7542b690b068352b000e41 is 50, key is test_row_0/C:col10/1733595374169/Put/seqid=0 2024-12-07T18:16:15,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742386_1562 (size=12001) 2024-12-07T18:16:15,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/953499dccd7542b690b068352b000e41 2024-12-07T18:16:15,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/82bc91843a5c4467afab88a308016b9a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/82bc91843a5c4467afab88a308016b9a 2024-12-07T18:16:15,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/82bc91843a5c4467afab88a308016b9a, entries=200, sequenceid=64, filesize=38.6 K 2024-12-07T18:16:15,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/1182147fc71e499191ff214fa8d06c99 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1182147fc71e499191ff214fa8d06c99 2024-12-07T18:16:15,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1182147fc71e499191ff214fa8d06c99, entries=150, sequenceid=64, filesize=11.7 K 2024-12-07T18:16:15,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/953499dccd7542b690b068352b000e41 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/953499dccd7542b690b068352b000e41 2024-12-07T18:16:15,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/953499dccd7542b690b068352b000e41, entries=150, sequenceid=64, filesize=11.7 K 2024-12-07T18:16:15,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ec33782b2c22c62c1e44260bfbcf6877 in 1256ms, sequenceid=64, compaction requested=true 2024-12-07T18:16:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:16:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:15,426 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:16:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:15,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:16:15,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:15,427 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:15,429 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:15,429 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:15,429 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/A is initiating minor compaction (all files) 2024-12-07T18:16:15,429 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/B is initiating minor compaction (all files) 2024-12-07T18:16:15,429 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/A in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:15,429 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/B in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:15,430 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/54c683fdd3f546aa84fd9f8c249bae52, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/47b0bb6dd7524cffb336decf1e4a48b3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1182147fc71e499191ff214fa8d06c99] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=35.2 K 2024-12-07T18:16:15,430 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/b431724f3e0f4b4fb39a2537de36c0f2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/2d94bf856b6b422093079756b4d4634e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/82bc91843a5c4467afab88a308016b9a] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=99.1 K 2024-12-07T18:16:15,430 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:15,430 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/b431724f3e0f4b4fb39a2537de36c0f2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/2d94bf856b6b422093079756b4d4634e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/82bc91843a5c4467afab88a308016b9a] 2024-12-07T18:16:15,430 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 54c683fdd3f546aa84fd9f8c249bae52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733595370814 2024-12-07T18:16:15,430 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting b431724f3e0f4b4fb39a2537de36c0f2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733595370814 2024-12-07T18:16:15,430 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 47b0bb6dd7524cffb336decf1e4a48b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733595370857 2024-12-07T18:16:15,430 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d94bf856b6b422093079756b4d4634e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733595370857 2024-12-07T18:16:15,430 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 1182147fc71e499191ff214fa8d06c99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595373028 2024-12-07T18:16:15,430 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82bc91843a5c4467afab88a308016b9a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595373028 2024-12-07T18:16:15,435 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:15,437 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412070965f218b00e4d4ca274807fc4c1c917_ec33782b2c22c62c1e44260bfbcf6877 store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:15,438 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#B#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:15,438 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/d92d9e6fcb0d4dcebfae813c29c81763 is 50, key is test_row_0/B:col10/1733595374169/Put/seqid=0 2024-12-07T18:16:15,439 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412070965f218b00e4d4ca274807fc4c1c917_ec33782b2c22c62c1e44260bfbcf6877, store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:15,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-07T18:16:15,439 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412070965f218b00e4d4ca274807fc4c1c917_ec33782b2c22c62c1e44260bfbcf6877 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:15,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:15,439 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T18:16:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:15,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742387_1563 (size=12104) 2024-12-07T18:16:15,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412079b9db62428694f29a9d2e1b09e388442_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595374195/Put/seqid=0 2024-12-07T18:16:15,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742388_1564 (size=4469) 2024-12-07T18:16:15,462 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/d92d9e6fcb0d4dcebfae813c29c81763 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d92d9e6fcb0d4dcebfae813c29c81763 2024-12-07T18:16:15,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742389_1565 (size=12154) 2024-12-07T18:16:15,466 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/B of ec33782b2c22c62c1e44260bfbcf6877 into d92d9e6fcb0d4dcebfae813c29c81763(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:15,466 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:15,466 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/B, priority=13, startTime=1733595375426; duration=0sec 2024-12-07T18:16:15,466 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:15,466 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:B 2024-12-07T18:16:15,466 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:15,467 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:15,467 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/C is initiating minor compaction (all files) 2024-12-07T18:16:15,467 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/C in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:15,468 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/94e04c56f9b749f39850420fe8399fcc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/c56f306406ca4fcf952004a99076ef3c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/953499dccd7542b690b068352b000e41] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=35.2 K 2024-12-07T18:16:15,468 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 94e04c56f9b749f39850420fe8399fcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733595370814 2024-12-07T18:16:15,468 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting c56f306406ca4fcf952004a99076ef3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733595370857 2024-12-07T18:16:15,468 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 953499dccd7542b690b068352b000e41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595373028 2024-12-07T18:16:15,475 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:15,475 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/0b23a87149cb4971a72723ec297c3549 is 50, key is test_row_0/C:col10/1733595374169/Put/seqid=0 2024-12-07T18:16:15,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742390_1566 (size=12104) 2024-12-07T18:16:15,482 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/0b23a87149cb4971a72723ec297c3549 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0b23a87149cb4971a72723ec297c3549 2024-12-07T18:16:15,486 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/C of ec33782b2c22c62c1e44260bfbcf6877 into 0b23a87149cb4971a72723ec297c3549(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:15,486 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:15,486 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/C, priority=13, startTime=1733595375427; duration=0sec 2024-12-07T18:16:15,486 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:15,486 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:C 2024-12-07T18:16:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-07T18:16:15,860 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#A#compaction#477 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:15,861 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/34683531c8874f5caf394a223b35de14 is 175, key is test_row_0/A:col10/1733595374169/Put/seqid=0 2024-12-07T18:16:15,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:15,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742391_1567 (size=31058) 2024-12-07T18:16:15,868 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412079b9db62428694f29a9d2e1b09e388442_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412079b9db62428694f29a9d2e1b09e388442_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:15,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/f975014e4ccf4b3486133172d5b5d7e9, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:15,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/f975014e4ccf4b3486133172d5b5d7e9 is 175, key is test_row_0/A:col10/1733595374195/Put/seqid=0 2024-12-07T18:16:15,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742392_1568 (size=30955) 2024-12-07T18:16:16,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-07T18:16:16,269 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/34683531c8874f5caf394a223b35de14 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/34683531c8874f5caf394a223b35de14 2024-12-07T18:16:16,272 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/f975014e4ccf4b3486133172d5b5d7e9 2024-12-07T18:16:16,274 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/A of ec33782b2c22c62c1e44260bfbcf6877 into 34683531c8874f5caf394a223b35de14(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:16,274 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:16,274 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/A, priority=13, startTime=1733595375426; duration=0sec 2024-12-07T18:16:16,274 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:16,274 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:A 2024-12-07T18:16:16,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/290926a746e542f886fe84bb8502a3e3 is 50, key is test_row_0/B:col10/1733595374195/Put/seqid=0 2024-12-07T18:16:16,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742393_1569 (size=12001) 2024-12-07T18:16:16,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:16,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:16,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:16,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595436371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:16,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595436374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:16,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595436374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:16,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:16,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595436478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:16,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595436489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595436489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:16,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595436688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:16,695 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/290926a746e542f886fe84bb8502a3e3 2024-12-07T18:16:16,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595436699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:16,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/3945541ed3d64e30843066aab2fc07a7 is 50, key is test_row_0/C:col10/1733595374195/Put/seqid=0 2024-12-07T18:16:16,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595436699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:16,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742394_1570 (size=12001) 2024-12-07T18:16:17,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595436998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595437008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595437008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,108 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/3945541ed3d64e30843066aab2fc07a7 2024-12-07T18:16:17,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/f975014e4ccf4b3486133172d5b5d7e9 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/f975014e4ccf4b3486133172d5b5d7e9 2024-12-07T18:16:17,114 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/f975014e4ccf4b3486133172d5b5d7e9, entries=150, sequenceid=82, filesize=30.2 K 2024-12-07T18:16:17,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/290926a746e542f886fe84bb8502a3e3 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/290926a746e542f886fe84bb8502a3e3 2024-12-07T18:16:17,118 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/290926a746e542f886fe84bb8502a3e3, entries=150, sequenceid=82, filesize=11.7 K 2024-12-07T18:16:17,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/3945541ed3d64e30843066aab2fc07a7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/3945541ed3d64e30843066aab2fc07a7 2024-12-07T18:16:17,122 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/3945541ed3d64e30843066aab2fc07a7, entries=150, sequenceid=82, filesize=11.7 K 2024-12-07T18:16:17,122 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ec33782b2c22c62c1e44260bfbcf6877 in 1683ms, sequenceid=82, compaction requested=false 2024-12-07T18:16:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-07T18:16:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-07T18:16:17,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-07T18:16:17,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1430 sec 2024-12-07T18:16:17,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.1470 sec 2024-12-07T18:16:17,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-07T18:16:17,128 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:17,129 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:16:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-07T18:16:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-07T18:16:17,130 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:16:17,131 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:16:17,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:16:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-07T18:16:17,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-07T18:16:17,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:17,283 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T18:16:17,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:17,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:17,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:17,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:17,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:17,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:17,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073c1396892be1401f94a22b2eaf131cc5_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595376373/Put/seqid=0 2024-12-07T18:16:17,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742395_1571 (size=12154) 2024-12-07T18:16:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:17,298 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073c1396892be1401f94a22b2eaf131cc5_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073c1396892be1401f94a22b2eaf131cc5_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/1dc00a78c5a64c30b7923aa3e723ef8b, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:17,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/1dc00a78c5a64c30b7923aa3e723ef8b is 175, key is test_row_0/A:col10/1733595376373/Put/seqid=0 2024-12-07T18:16:17,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742396_1572 (size=30955) 2024-12-07T18:16:17,304 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/1dc00a78c5a64c30b7923aa3e723ef8b 2024-12-07T18:16:17,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/d40a825114df49f6be3d075e1bdd4604 is 50, key is test_row_0/B:col10/1733595376373/Put/seqid=0 2024-12-07T18:16:17,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742397_1573 (size=12001) 2024-12-07T18:16:17,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-07T18:16:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:17,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:17,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595437537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595437538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,543 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595437540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595437649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595437649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595437649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,720 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/d40a825114df49f6be3d075e1bdd4604 2024-12-07T18:16:17,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/4c9ef0ecbe35462e81425bf84e89c8bf is 50, key is test_row_0/C:col10/1733595376373/Put/seqid=0 2024-12-07T18:16:17,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742398_1574 (size=12001) 2024-12-07T18:16:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-07T18:16:17,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595437859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595437859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595437859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,131 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/4c9ef0ecbe35462e81425bf84e89c8bf 2024-12-07T18:16:18,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/1dc00a78c5a64c30b7923aa3e723ef8b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1dc00a78c5a64c30b7923aa3e723ef8b 2024-12-07T18:16:18,139 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1dc00a78c5a64c30b7923aa3e723ef8b, entries=150, sequenceid=99, filesize=30.2 K 2024-12-07T18:16:18,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/d40a825114df49f6be3d075e1bdd4604 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d40a825114df49f6be3d075e1bdd4604 2024-12-07T18:16:18,143 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d40a825114df49f6be3d075e1bdd4604, entries=150, sequenceid=99, filesize=11.7 K 2024-12-07T18:16:18,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/4c9ef0ecbe35462e81425bf84e89c8bf as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/4c9ef0ecbe35462e81425bf84e89c8bf 2024-12-07T18:16:18,146 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/4c9ef0ecbe35462e81425bf84e89c8bf, entries=150, sequenceid=99, filesize=11.7 K 2024-12-07T18:16:18,147 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ec33782b2c22c62c1e44260bfbcf6877 in 864ms, sequenceid=99, compaction requested=true 2024-12-07T18:16:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:18,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-07T18:16:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-07T18:16:18,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-07T18:16:18,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-12-07T18:16:18,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.0210 sec 2024-12-07T18:16:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:18,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-07T18:16:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:18,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120703e0c395a2c3498a885f5393e4fc8059_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595377538/Put/seqid=0 2024-12-07T18:16:18,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742399_1575 (size=14594) 2024-12-07T18:16:18,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595438185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595438186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595438191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-07T18:16:18,269 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:18,269 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:16:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-07T18:16:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T18:16:18,271 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:16:18,271 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:16:18,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:16:18,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595438299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595438299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595438308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T18:16:18,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T18:16:18,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:18,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:18,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:18,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595438508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595438508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595438518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T18:16:18,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:18,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,580 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:18,583 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120703e0c395a2c3498a885f5393e4fc8059_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120703e0c395a2c3498a885f5393e4fc8059_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:18,584 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/42e03efa5a5e40f9b010de6c07ffecb4, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:18,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/42e03efa5a5e40f9b010de6c07ffecb4 is 175, key is test_row_0/A:col10/1733595377538/Put/seqid=0 2024-12-07T18:16:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T18:16:18,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742400_1576 (size=39549) 2024-12-07T18:16:18,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T18:16:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:18,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:18,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595438818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595438819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:18,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595438828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:18,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T18:16:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:18,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:18,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T18:16:18,989 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/42e03efa5a5e40f9b010de6c07ffecb4 2024-12-07T18:16:18,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/e9499df93b444b3ca740155825f91d4d is 50, key is test_row_0/B:col10/1733595377538/Put/seqid=0 2024-12-07T18:16:18,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742401_1577 (size=12001) 2024-12-07T18:16:19,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T18:16:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:19,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:19,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46788 deadline: 1733595439068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:19,072 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in ec33782b2c22c62c1e44260bfbcf6877 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:19,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:19,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46802 deadline: 1733595439098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:19,102 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in ec33782b2c22c62c1e44260bfbcf6877 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:19,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:19,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T18:16:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:19,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:19,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595439328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:19,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:19,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595439328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:19,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T18:16:19,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:19,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:19,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:19,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595439339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:19,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/e9499df93b444b3ca740155825f91d4d 2024-12-07T18:16:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T18:16:19,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/9bf7010c6cb84383bdf9f302ba7bd95e is 50, key is test_row_0/C:col10/1733595377538/Put/seqid=0 2024-12-07T18:16:19,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742402_1578 (size=12001) 2024-12-07T18:16:19,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/9bf7010c6cb84383bdf9f302ba7bd95e 2024-12-07T18:16:19,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/42e03efa5a5e40f9b010de6c07ffecb4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/42e03efa5a5e40f9b010de6c07ffecb4 2024-12-07T18:16:19,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/42e03efa5a5e40f9b010de6c07ffecb4, entries=200, sequenceid=118, filesize=38.6 K 2024-12-07T18:16:19,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/e9499df93b444b3ca740155825f91d4d as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/e9499df93b444b3ca740155825f91d4d 2024-12-07T18:16:19,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/e9499df93b444b3ca740155825f91d4d, entries=150, sequenceid=118, filesize=11.7 K 2024-12-07T18:16:19,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/9bf7010c6cb84383bdf9f302ba7bd95e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/9bf7010c6cb84383bdf9f302ba7bd95e 2024-12-07T18:16:19,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/9bf7010c6cb84383bdf9f302ba7bd95e, entries=150, sequenceid=118, filesize=11.7 K 2024-12-07T18:16:19,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ec33782b2c22c62c1e44260bfbcf6877 in 1277ms, sequenceid=118, compaction requested=true 2024-12-07T18:16:19,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:19,447 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:16:19,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:16:19,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:19,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:16:19,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:19,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:16:19,448 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:16:19,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:19,448 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:16:19,448 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/A is initiating minor compaction (all files) 2024-12-07T18:16:19,448 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/A in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,449 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/34683531c8874f5caf394a223b35de14, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/f975014e4ccf4b3486133172d5b5d7e9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1dc00a78c5a64c30b7923aa3e723ef8b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/42e03efa5a5e40f9b010de6c07ffecb4] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=129.4 K 2024-12-07T18:16:19,449 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,449 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/34683531c8874f5caf394a223b35de14, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/f975014e4ccf4b3486133172d5b5d7e9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1dc00a78c5a64c30b7923aa3e723ef8b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/42e03efa5a5e40f9b010de6c07ffecb4] 2024-12-07T18:16:19,449 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34683531c8874f5caf394a223b35de14, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595373028 2024-12-07T18:16:19,450 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting f975014e4ccf4b3486133172d5b5d7e9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733595374190 2024-12-07T18:16:19,450 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:16:19,450 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/B is initiating minor compaction (all files) 2024-12-07T18:16:19,450 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/B in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,450 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d92d9e6fcb0d4dcebfae813c29c81763, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/290926a746e542f886fe84bb8502a3e3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d40a825114df49f6be3d075e1bdd4604, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/e9499df93b444b3ca740155825f91d4d] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=47.0 K 2024-12-07T18:16:19,450 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dc00a78c5a64c30b7923aa3e723ef8b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733595376366 2024-12-07T18:16:19,450 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting d92d9e6fcb0d4dcebfae813c29c81763, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595373028 2024-12-07T18:16:19,451 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42e03efa5a5e40f9b010de6c07ffecb4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733595377532 2024-12-07T18:16:19,451 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 290926a746e542f886fe84bb8502a3e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733595374190 2024-12-07T18:16:19,451 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting d40a825114df49f6be3d075e1bdd4604, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733595376366 2024-12-07T18:16:19,451 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e9499df93b444b3ca740155825f91d4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733595377537 2024-12-07T18:16:19,457 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:19,461 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412074fe1037b751d4d02b8556acbd05e005b_ec33782b2c22c62c1e44260bfbcf6877 store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:19,461 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#B#compaction#490 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:19,463 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/1af3000bd4664018bca832f8dcf144b8 is 50, key is test_row_0/B:col10/1733595377538/Put/seqid=0 2024-12-07T18:16:19,464 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412074fe1037b751d4d02b8556acbd05e005b_ec33782b2c22c62c1e44260bfbcf6877, store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:19,464 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412074fe1037b751d4d02b8556acbd05e005b_ec33782b2c22c62c1e44260bfbcf6877 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:19,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742403_1579 (size=12241) 2024-12-07T18:16:19,480 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/1af3000bd4664018bca832f8dcf144b8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1af3000bd4664018bca832f8dcf144b8 2024-12-07T18:16:19,490 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/B of ec33782b2c22c62c1e44260bfbcf6877 into 1af3000bd4664018bca832f8dcf144b8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:19,490 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:19,490 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/B, priority=12, startTime=1733595379447; duration=0sec 2024-12-07T18:16:19,490 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:19,490 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:B 2024-12-07T18:16:19,490 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:16:19,491 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:16:19,491 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/C is initiating minor compaction (all files) 2024-12-07T18:16:19,491 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/C in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,491 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0b23a87149cb4971a72723ec297c3549, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/3945541ed3d64e30843066aab2fc07a7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/4c9ef0ecbe35462e81425bf84e89c8bf, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/9bf7010c6cb84383bdf9f302ba7bd95e] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=47.0 K 2024-12-07T18:16:19,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742404_1580 (size=4469) 2024-12-07T18:16:19,492 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b23a87149cb4971a72723ec297c3549, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733595373028 2024-12-07T18:16:19,493 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 3945541ed3d64e30843066aab2fc07a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733595374190 2024-12-07T18:16:19,494 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c9ef0ecbe35462e81425bf84e89c8bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733595376366 2024-12-07T18:16:19,494 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf7010c6cb84383bdf9f302ba7bd95e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733595377537 2024-12-07T18:16:19,495 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#A#compaction#489 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:19,495 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/fce1d0c097da4f3888c14016081a5025 is 175, key is test_row_0/A:col10/1733595377538/Put/seqid=0 2024-12-07T18:16:19,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-07T18:16:19,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:19,496 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-07T18:16:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:19,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:19,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742405_1581 (size=31195) 2024-12-07T18:16:19,509 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/fce1d0c097da4f3888c14016081a5025 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/fce1d0c097da4f3888c14016081a5025 2024-12-07T18:16:19,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207c322abe5f37543a991a3f34fc915aabb_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595378183/Put/seqid=0 2024-12-07T18:16:19,511 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#C#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:19,512 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/18dbf76e353e4857a1835413aed50aee is 50, key is test_row_0/C:col10/1733595377538/Put/seqid=0 2024-12-07T18:16:19,520 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/A of ec33782b2c22c62c1e44260bfbcf6877 into fce1d0c097da4f3888c14016081a5025(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:19,520 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:19,520 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/A, priority=12, startTime=1733595379447; duration=0sec 2024-12-07T18:16:19,520 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:19,520 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:A 2024-12-07T18:16:19,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742407_1583 (size=12241) 2024-12-07T18:16:19,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742406_1582 (size=12204) 2024-12-07T18:16:19,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:19,928 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/18dbf76e353e4857a1835413aed50aee as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/18dbf76e353e4857a1835413aed50aee 2024-12-07T18:16:19,930 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207c322abe5f37543a991a3f34fc915aabb_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c322abe5f37543a991a3f34fc915aabb_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:19,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/452074764b13421ea53801fc574e8fa5, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:19,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/452074764b13421ea53801fc574e8fa5 is 175, key is test_row_0/A:col10/1733595378183/Put/seqid=0 2024-12-07T18:16:19,933 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/C of ec33782b2c22c62c1e44260bfbcf6877 into 18dbf76e353e4857a1835413aed50aee(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:19,933 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:19,933 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/C, priority=12, startTime=1733595379447; duration=0sec 2024-12-07T18:16:19,933 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:19,933 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:C 2024-12-07T18:16:19,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742408_1584 (size=31005) 2024-12-07T18:16:20,337 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/452074764b13421ea53801fc574e8fa5 2024-12-07T18:16:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:20,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:20,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/9110d01ee01d4ae288bf36532ff5a451 is 50, key is test_row_0/B:col10/1733595378183/Put/seqid=0 2024-12-07T18:16:20,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742409_1585 (size=12051) 2024-12-07T18:16:20,349 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/9110d01ee01d4ae288bf36532ff5a451 2024-12-07T18:16:20,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/d0c494f81f224526befe48967e9e5322 is 50, key is test_row_0/C:col10/1733595378183/Put/seqid=0 2024-12-07T18:16:20,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742410_1586 (size=12051) 2024-12-07T18:16:20,362 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/d0c494f81f224526befe48967e9e5322 2024-12-07T18:16:20,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/452074764b13421ea53801fc574e8fa5 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/452074764b13421ea53801fc574e8fa5 2024-12-07T18:16:20,370 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/452074764b13421ea53801fc574e8fa5, entries=150, sequenceid=132, filesize=30.3 K 2024-12-07T18:16:20,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/9110d01ee01d4ae288bf36532ff5a451 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/9110d01ee01d4ae288bf36532ff5a451 2024-12-07T18:16:20,376 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/9110d01ee01d4ae288bf36532ff5a451, entries=150, sequenceid=132, filesize=11.8 K 2024-12-07T18:16:20,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/d0c494f81f224526befe48967e9e5322 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d0c494f81f224526befe48967e9e5322 2024-12-07T18:16:20,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:20,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595440373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:20,381 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d0c494f81f224526befe48967e9e5322, entries=150, sequenceid=132, filesize=11.8 K 2024-12-07T18:16:20,382 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ec33782b2c22c62c1e44260bfbcf6877 in 886ms, sequenceid=132, compaction requested=false 2024-12-07T18:16:20,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:20,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:20,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-07T18:16:20,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-07T18:16:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:20,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-07T18:16:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595380370, max timestamp=1733595380383Num uniques 859; 2024-12-07T18:16:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595380370, max timestamp=1733595380383Num uniques 859; 2024-12-07T18:16:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:16:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:16:20,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733595380370, max timestamp=1733595380383Num uniques 859; 2024-12-07T18:16:20,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-07T18:16:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:20,387 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:20,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:20,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:20,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:20,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-07T18:16:20,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1160 sec 2024-12-07T18:16:20,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1260 sec 2024-12-07T18:16:20,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207be1c5c747ac64014953c8dfd22c9aa26_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595380384/Put/seqid=0 2024-12-07T18:16:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-07T18:16:20,419 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:20,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742411_1587 (size=14794) 2024-12-07T18:16:20,419 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:20,420 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:16:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-07T18:16:20,421 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:16:20,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T18:16:20,422 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:16:20,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:16:20,423 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207be1c5c747ac64014953c8dfd22c9aa26_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207be1c5c747ac64014953c8dfd22c9aa26_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:20,424 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/1b32b9041ad642bfa530ef3f4c97fefa, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:20,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/1b32b9041ad642bfa530ef3f4c97fefa is 175, key is test_row_0/A:col10/1733595380384/Put/seqid=0 2024-12-07T18:16:20,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742412_1588 (size=39749) 2024-12-07T18:16:20,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/1b32b9041ad642bfa530ef3f4c97fefa 2024-12-07T18:16:20,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/11fc5c7355624a9fbb8e0a18bfcdb7df is 50, key is test_row_0/B:col10/1733595380384/Put/seqid=0 2024-12-07T18:16:20,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742413_1589 (size=12151) 2024-12-07T18:16:20,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595380426, max timestamp=1733595380441Num uniques 859; 2024-12-07T18:16:20,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595380426, max timestamp=1733595380441Num uniques 859; 2024-12-07T18:16:20,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:16:20,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:16:20,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595380426, max timestamp=1733595380441Num uniques 859; 2024-12-07T18:16:20,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:16:20,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733595380477, max timestamp=1733595380519Num uniques 1068; 2024-12-07T18:16:20,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-07T18:16:20,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733595380477, max timestamp=1733595380519Num uniques 1068; 2024-12-07T18:16:20,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733595380477, max timestamp=1733595380519Num uniques 1068; 2024-12-07T18:16:20,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-07T18:16:20,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-07T18:16:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T18:16:20,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:20,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733595380548, max timestamp=1733595380564Num uniques 1127; 2024-12-07T18:16:20,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733595380548, max timestamp=1733595380564Num uniques 1127; 2024-12-07T18:16:20,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-07T18:16:20,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-07T18:16:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595440559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:20,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733595380548, max timestamp=1733595380564Num uniques 1127; 2024-12-07T18:16:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:20,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-12-07T18:16:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:20,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T18:16:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:20,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:20,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:20,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733595380614, max timestamp=1733595380624Num uniques 1036; 2024-12-07T18:16:20,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-07T18:16:20,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733595380614, max timestamp=1733595380624Num uniques 1036; 2024-12-07T18:16:20,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-07T18:16:20,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733595380614, max timestamp=1733595380624Num uniques 1036; 2024-12-07T18:16:20,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-07T18:16:20,684 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733595380653, max timestamp=1733595380680Num uniques 1045; 2024-12-07T18:16:20,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-07T18:16:20,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733595380653, max timestamp=1733595380680Num uniques 1045; 2024-12-07T18:16:20,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-07T18:16:20,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733595380653, max timestamp=1733595380680Num uniques 1045; 2024-12-07T18:16:20,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-07T18:16:20,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T18:16:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:20,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:20,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:20,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733595380715, max timestamp=1733595380724Num uniques 1054; 2024-12-07T18:16:20,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-07T18:16:20,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733595380715, max timestamp=1733595380724Num uniques 1054; 2024-12-07T18:16:20,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-07T18:16:20,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733595380715, max timestamp=1733595380724Num uniques 1054; 2024-12-07T18:16:20,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-07T18:16:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T18:16:20,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595440760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:20,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595440762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:20,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733595380747, max timestamp=1733595380766Num uniques 1113; 2024-12-07T18:16:20,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-12-07T18:16:20,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733595380747, max timestamp=1733595380766Num uniques 1113; 2024-12-07T18:16:20,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-12-07T18:16:20,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1733595380747, max timestamp=1733595380766Num uniques 1113; 2024-12-07T18:16:20,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=7458, suffix heap size=143720,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=163 2024-12-07T18:16:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:20,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/11fc5c7355624a9fbb8e0a18bfcdb7df 2024-12-07T18:16:20,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/69799728d097445cb7afabb8da908c26 is 50, key is test_row_0/C:col10/1733595380384/Put/seqid=0 2024-12-07T18:16:20,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742414_1590 (size=12151) 2024-12-07T18:16:20,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T18:16:20,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:20,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:20,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:20,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:20,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:20,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733595380840, max timestamp=1733595380876Num uniques 1022; 2024-12-07T18:16:20,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-07T18:16:20,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733595380840, max timestamp=1733595380876Num uniques 1022; 2024-12-07T18:16:20,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-07T18:16:20,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733595380840, max timestamp=1733595380876Num uniques 1022; 2024-12-07T18:16:20,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-12-07T18:16:20,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:20,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733595380900, max timestamp=1733595380921Num uniques 1031; 2024-12-07T18:16:20,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-07T18:16:20,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733595380900, max timestamp=1733595380921Num uniques 1031; 2024-12-07T18:16:20,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-07T18:16:20,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733595380900, max timestamp=1733595380921Num uniques 1031; 2024-12-07T18:16:20,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-07T18:16:20,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733595380947, max timestamp=1733595380957Num uniques 1040; 2024-12-07T18:16:20,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-07T18:16:20,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733595380947, max timestamp=1733595380957Num uniques 1040; 2024-12-07T18:16:20,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-07T18:16:20,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733595380947, max timestamp=1733595380957Num uniques 1040; 2024-12-07T18:16:20,962 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-07T18:16:21,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733595380979, max timestamp=1733595381000Num uniques 1049; 2024-12-07T18:16:21,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733595380979, max timestamp=1733595381000Num uniques 1049; 2024-12-07T18:16:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-07T18:16:21,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-07T18:16:21,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1733595380979, max timestamp=1733595381000Num uniques 1049; 2024-12-07T18:16:21,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-12-07T18:16:21,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T18:16:21,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:21,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:21,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:21,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:21,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595441040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733595381027, max timestamp=1733595381043Num uniques 1108; 2024-12-07T18:16:21,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-12-07T18:16:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T18:16:21,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733595381027, max timestamp=1733595381043Num uniques 1108; 2024-12-07T18:16:21,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-12-07T18:16:21,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733595381027, max timestamp=1733595381043Num uniques 1108; 2024-12-07T18:16:21,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-12-07T18:16:21,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733595381086, max timestamp=1733595381100Num uniques 1067; 2024-12-07T18:16:21,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-12-07T18:16:21,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733595381086, max timestamp=1733595381100Num uniques 1067; 2024-12-07T18:16:21,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-12-07T18:16:21,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733595381086, max timestamp=1733595381100Num uniques 1067; 2024-12-07T18:16:21,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=9932, suffix heap size=138936,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=217 2024-12-07T18:16:21,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733595381146, max timestamp=1733595381157Num uniques 1076; 2024-12-07T18:16:21,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733595381146, max timestamp=1733595381157Num uniques 1076; 2024-12-07T18:16:21,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-07T18:16:21,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-07T18:16:21,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1733595381146, max timestamp=1733595381157Num uniques 1076; 2024-12-07T18:16:21,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=8056, suffix heap size=139872,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=176 2024-12-07T18:16:21,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T18:16:21,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:21,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:21,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:21,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:21,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733595381182, max timestamp=1733595381198Num uniques 1035; 2024-12-07T18:16:21,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=13050, suffix heap size=135608,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=285 2024-12-07T18:16:21,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733595381182, max timestamp=1733595381198Num uniques 1035; 2024-12-07T18:16:21,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=13050, suffix heap size=135608,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=285 2024-12-07T18:16:21,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1733595381182, max timestamp=1733595381198Num uniques 1035; 2024-12-07T18:16:21,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=13050, suffix heap size=135608,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=285 2024-12-07T18:16:21,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/69799728d097445cb7afabb8da908c26 2024-12-07T18:16:21,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/1b32b9041ad642bfa530ef3f4c97fefa as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1b32b9041ad642bfa530ef3f4c97fefa 2024-12-07T18:16:21,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1b32b9041ad642bfa530ef3f4c97fefa, entries=200, sequenceid=156, filesize=38.8 K 2024-12-07T18:16:21,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/11fc5c7355624a9fbb8e0a18bfcdb7df as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/11fc5c7355624a9fbb8e0a18bfcdb7df 2024-12-07T18:16:21,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/11fc5c7355624a9fbb8e0a18bfcdb7df, entries=150, sequenceid=156, filesize=11.9 K 2024-12-07T18:16:21,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/69799728d097445cb7afabb8da908c26 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/69799728d097445cb7afabb8da908c26 2024-12-07T18:16:21,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/69799728d097445cb7afabb8da908c26, entries=150, sequenceid=156, filesize=11.9 K 2024-12-07T18:16:21,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=147.60 KB/151140 for ec33782b2c22c62c1e44260bfbcf6877 in 885ms, sequenceid=156, compaction requested=true 2024-12-07T18:16:21,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:21,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:16:21,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:21,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:16:21,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:21,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:16:21,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:21,269 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:21,269 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:21,270 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:21,270 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/B is initiating minor compaction (all files) 2024-12-07T18:16:21,270 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/B in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,270 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1af3000bd4664018bca832f8dcf144b8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/9110d01ee01d4ae288bf36532ff5a451, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/11fc5c7355624a9fbb8e0a18bfcdb7df] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=35.6 K 2024-12-07T18:16:21,271 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101949 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:21,271 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/A is initiating minor compaction (all files) 2024-12-07T18:16:21,271 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/A in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,271 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/fce1d0c097da4f3888c14016081a5025, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/452074764b13421ea53801fc574e8fa5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1b32b9041ad642bfa530ef3f4c97fefa] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=99.6 K 2024-12-07T18:16:21,271 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,271 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/fce1d0c097da4f3888c14016081a5025, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/452074764b13421ea53801fc574e8fa5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1b32b9041ad642bfa530ef3f4c97fefa] 2024-12-07T18:16:21,271 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 1af3000bd4664018bca832f8dcf144b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733595377537 2024-12-07T18:16:21,271 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting fce1d0c097da4f3888c14016081a5025, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733595377537 2024-12-07T18:16:21,271 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 9110d01ee01d4ae288bf36532ff5a451, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733595378175 2024-12-07T18:16:21,272 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 452074764b13421ea53801fc574e8fa5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733595378175 2024-12-07T18:16:21,272 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 11fc5c7355624a9fbb8e0a18bfcdb7df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733595380371 2024-12-07T18:16:21,272 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b32b9041ad642bfa530ef3f4c97fefa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733595380370 2024-12-07T18:16:21,277 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:21,278 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#B#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:21,278 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412074bd26f6f5ba945d1b4887413d6d350f6_ec33782b2c22c62c1e44260bfbcf6877 store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:21,279 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/c7221637dab6487a8d577fbf7b17d630 is 50, key is test_row_0/B:col10/1733595380384/Put/seqid=0 2024-12-07T18:16:21,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8874, totalHeapSize=20416, min timestamp=1733595381224, max timestamp=1733595381275Num uniques 1144; 2024-12-07T18:16:21,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8874, totalHeapSize=20416, min timestamp=1733595381224, max timestamp=1733595381275Num uniques 1144; 2024-12-07T18:16:21,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52384, new segment data size=8874, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-12-07T18:16:21,280 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412074bd26f6f5ba945d1b4887413d6d350f6_ec33782b2c22c62c1e44260bfbcf6877, store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:21,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52384, new segment data size=8874, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-12-07T18:16:21,280 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412074bd26f6f5ba945d1b4887413d6d350f6_ec33782b2c22c62c1e44260bfbcf6877 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:21,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=194, cellSize=8874, totalHeapSize=20416, min timestamp=1733595381224, max timestamp=1733595381275Num uniques 1144; 2024-12-07T18:16:21,280 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52384, new segment data size=8874, suffix heap size=146944,new segment heap size=20416 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=194 2024-12-07T18:16:21,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742415_1591 (size=12493) 2024-12-07T18:16:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:21,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742416_1592 (size=4469) 2024-12-07T18:16:21,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=60.38 KB heapSize=162.56 KB 2024-12-07T18:16:21,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:21,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:21,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:21,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:21,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:21,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:21,289 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#A#compaction#499 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:21,289 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/49b69bec64f64b5c93b8275d93288e6e is 175, key is test_row_0/A:col10/1733595380384/Put/seqid=0 2024-12-07T18:16:21,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742417_1593 (size=31447) 2024-12-07T18:16:21,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207644066a5187a4d2c82cdde650bc4bf18_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595381224/Put/seqid=0 2024-12-07T18:16:21,301 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/49b69bec64f64b5c93b8275d93288e6e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/49b69bec64f64b5c93b8275d93288e6e 2024-12-07T18:16:21,307 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/A of ec33782b2c22c62c1e44260bfbcf6877 into 49b69bec64f64b5c93b8275d93288e6e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:21,307 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:21,307 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/A, priority=13, startTime=1733595381269; duration=0sec 2024-12-07T18:16:21,307 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:21,307 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:A 2024-12-07T18:16:21,307 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-07T18:16:21,308 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-07T18:16:21,308 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/C is initiating minor compaction (all files) 2024-12-07T18:16:21,308 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/C in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,308 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/18dbf76e353e4857a1835413aed50aee, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d0c494f81f224526befe48967e9e5322, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/69799728d097445cb7afabb8da908c26] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=35.6 K 2024-12-07T18:16:21,309 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18dbf76e353e4857a1835413aed50aee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733595377537 2024-12-07T18:16:21,310 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0c494f81f224526befe48967e9e5322, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1733595378175 2024-12-07T18:16:21,310 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69799728d097445cb7afabb8da908c26, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733595380371 2024-12-07T18:16:21,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742418_1594 (size=12454) 2024-12-07T18:16:21,331 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:21,337 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207644066a5187a4d2c82cdde650bc4bf18_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207644066a5187a4d2c82cdde650bc4bf18_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:21,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733595381317, max timestamp=1733595381335Num uniques 859; 2024-12-07T18:16:21,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-07T18:16:21,339 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/3666ab865fa74fb682f0ff3f0eb12315, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:21,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T18:16:21,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:21,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/3666ab865fa74fb682f0ff3f0eb12315 is 175, key is test_row_0/A:col10/1733595381224/Put/seqid=0 2024-12-07T18:16:21,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:21,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T18:16:21,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595381324, max timestamp=1733595381335Num uniques 859; 2024-12-07T18:16:21,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:16:21,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733595381324, max timestamp=1733595381335Num uniques 859; 2024-12-07T18:16:21,343 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/92acd7e2389b:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-07T18:16:21,346 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#C#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:21,347 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/d4df26893bb449a094e75b92e739ea50 is 50, key is test_row_0/C:col10/1733595380384/Put/seqid=0 2024-12-07T18:16:21,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742420_1596 (size=12493) 2024-12-07T18:16:21,357 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/d4df26893bb449a094e75b92e739ea50 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d4df26893bb449a094e75b92e739ea50 2024-12-07T18:16:21,362 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/C of ec33782b2c22c62c1e44260bfbcf6877 into d4df26893bb449a094e75b92e739ea50(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:21,362 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:21,362 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/C, priority=13, startTime=1733595381269; duration=0sec 2024-12-07T18:16:21,363 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:21,363 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:C 2024-12-07T18:16:21,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742419_1595 (size=31255) 2024-12-07T18:16:21,367 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/3666ab865fa74fb682f0ff3f0eb12315 2024-12-07T18:16:21,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/8bdd4211bd9040abbfffd2b9ad4d6396 is 50, key is test_row_0/B:col10/1733595381224/Put/seqid=0 2024-12-07T18:16:21,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742421_1597 (size=12301) 2024-12-07T18:16:21,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/8bdd4211bd9040abbfffd2b9ad4d6396 2024-12-07T18:16:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595441381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/12b58fe369ac4ff2839a8eb4173ac7ba is 50, key is test_row_0/C:col10/1733595381224/Put/seqid=0 2024-12-07T18:16:21,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595441386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595441388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742422_1598 (size=12301) 2024-12-07T18:16:21,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/12b58fe369ac4ff2839a8eb4173ac7ba 2024-12-07T18:16:21,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/3666ab865fa74fb682f0ff3f0eb12315 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/3666ab865fa74fb682f0ff3f0eb12315 2024-12-07T18:16:21,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/3666ab865fa74fb682f0ff3f0eb12315, entries=150, sequenceid=439, filesize=30.5 K 2024-12-07T18:16:21,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/8bdd4211bd9040abbfffd2b9ad4d6396 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/8bdd4211bd9040abbfffd2b9ad4d6396 2024-12-07T18:16:21,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/8bdd4211bd9040abbfffd2b9ad4d6396, entries=150, sequenceid=439, filesize=12.0 K 2024-12-07T18:16:21,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/12b58fe369ac4ff2839a8eb4173ac7ba as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/12b58fe369ac4ff2839a8eb4173ac7ba 2024-12-07T18:16:21,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/12b58fe369ac4ff2839a8eb4173ac7ba, entries=150, sequenceid=439, filesize=12.0 K 2024-12-07T18:16:21,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~162.52 KB/166416, currentSize=129.71 KB/132820 for ec33782b2c22c62c1e44260bfbcf6877 in 136ms, sequenceid=439, compaction requested=false 2024-12-07T18:16:21,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:21,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-07T18:16:21,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:21,493 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-12-07T18:16:21,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:21,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:21,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:21,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:21,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:21,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-07T18:16:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:21,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:21,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a2b6a0688c124e72b87d8a8ea03aa1c3_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595381379/Put/seqid=0 2024-12-07T18:16:21,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742423_1599 (size=12454) 2024-12-07T18:16:21,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595441510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595441511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,519 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595441516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T18:16:21,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595441628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595441628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595441628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,632 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,687 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/c7221637dab6487a8d577fbf7b17d630 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/c7221637dab6487a8d577fbf7b17d630 2024-12-07T18:16:21,690 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/B of ec33782b2c22c62c1e44260bfbcf6877 into c7221637dab6487a8d577fbf7b17d630(size=12.2 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:21,690 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:21,690 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/B, priority=13, startTime=1733595381269; duration=0sec 2024-12-07T18:16:21,690 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:21,690 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:B 2024-12-07T18:16:21,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595441838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595441838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595441838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:21,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:21,908 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207a2b6a0688c124e72b87d8a8ea03aa1c3_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a2b6a0688c124e72b87d8a8ea03aa1c3_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:21,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/a8d4f3f1ae404b31ba1b9c0932c4b442, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:21,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/a8d4f3f1ae404b31ba1b9c0932c4b442 is 175, key is test_row_0/A:col10/1733595381379/Put/seqid=0 2024-12-07T18:16:21,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742424_1600 (size=31255) 2024-12-07T18:16:22,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595442148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:22,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595442149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:22,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595442149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:22,314 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=476, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/a8d4f3f1ae404b31ba1b9c0932c4b442 2024-12-07T18:16:22,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/620ea3488df8445bbdf6fd4ca9b73b95 is 50, key is test_row_0/B:col10/1733595381379/Put/seqid=0 2024-12-07T18:16:22,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742425_1601 (size=12301) 2024-12-07T18:16:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T18:16:22,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595442659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:22,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595442659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:22,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595442659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:22,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:22,724 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/620ea3488df8445bbdf6fd4ca9b73b95 2024-12-07T18:16:22,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/7b71aaeb6e734b14b2ed7afb54b6db9b is 50, key is test_row_0/C:col10/1733595381379/Put/seqid=0 2024-12-07T18:16:22,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742426_1602 (size=12301) 2024-12-07T18:16:23,135 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/7b71aaeb6e734b14b2ed7afb54b6db9b 2024-12-07T18:16:23,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/a8d4f3f1ae404b31ba1b9c0932c4b442 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/a8d4f3f1ae404b31ba1b9c0932c4b442 2024-12-07T18:16:23,142 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/a8d4f3f1ae404b31ba1b9c0932c4b442, entries=150, sequenceid=476, filesize=30.5 K 2024-12-07T18:16:23,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/620ea3488df8445bbdf6fd4ca9b73b95 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/620ea3488df8445bbdf6fd4ca9b73b95 2024-12-07T18:16:23,145 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/620ea3488df8445bbdf6fd4ca9b73b95, entries=150, sequenceid=476, filesize=12.0 K 2024-12-07T18:16:23,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/7b71aaeb6e734b14b2ed7afb54b6db9b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/7b71aaeb6e734b14b2ed7afb54b6db9b 2024-12-07T18:16:23,148 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/7b71aaeb6e734b14b2ed7afb54b6db9b, entries=150, sequenceid=476, filesize=12.0 K 2024-12-07T18:16:23,149 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=53.67 KB/54960 for ec33782b2c22c62c1e44260bfbcf6877 in 1656ms, sequenceid=476, compaction requested=true 2024-12-07T18:16:23,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:23,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:23,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-07T18:16:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-07T18:16:23,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-07T18:16:23,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7280 sec 2024-12-07T18:16:23,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.7330 sec 2024-12-07T18:16:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:23,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-07T18:16:23,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:23,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:23,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:23,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:23,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:23,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:23,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412071e304453ad374feaa1938fc67b2c782e_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595383681/Put/seqid=0 2024-12-07T18:16:23,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742427_1603 (size=14994) 2024-12-07T18:16:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595443711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595443713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595443713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:23,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595443818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:23,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595443828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:23,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595443828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595444028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595444039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595444039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:24,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412071e304453ad374feaa1938fc67b2c782e_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071e304453ad374feaa1938fc67b2c782e_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:24,099 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/20c19b2376dc4196b3b3165ec36c0bc2, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:24,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/20c19b2376dc4196b3b3165ec36c0bc2 is 175, key is test_row_0/A:col10/1733595383681/Put/seqid=0 2024-12-07T18:16:24,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742428_1604 (size=39949) 2024-12-07T18:16:24,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595444338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595444348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595444348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,503 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=491, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/20c19b2376dc4196b3b3165ec36c0bc2 2024-12-07T18:16:24,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/93fbe0d46f844defbace3e98cc88ffa0 is 50, key is test_row_0/B:col10/1733595383681/Put/seqid=0 2024-12-07T18:16:24,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742429_1605 (size=12301) 2024-12-07T18:16:24,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/93fbe0d46f844defbace3e98cc88ffa0 2024-12-07T18:16:24,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/0813ec8990ba447c968f833ed3b21d2a is 50, key is test_row_0/C:col10/1733595383681/Put/seqid=0 2024-12-07T18:16:24,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742430_1606 (size=12301) 2024-12-07T18:16:24,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/0813ec8990ba447c968f833ed3b21d2a 2024-12-07T18:16:24,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/20c19b2376dc4196b3b3165ec36c0bc2 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/20c19b2376dc4196b3b3165ec36c0bc2 2024-12-07T18:16:24,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/20c19b2376dc4196b3b3165ec36c0bc2, entries=200, sequenceid=491, filesize=39.0 K 2024-12-07T18:16:24,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/93fbe0d46f844defbace3e98cc88ffa0 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/93fbe0d46f844defbace3e98cc88ffa0 2024-12-07T18:16:24,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/93fbe0d46f844defbace3e98cc88ffa0, entries=150, sequenceid=491, filesize=12.0 K 2024-12-07T18:16:24,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/0813ec8990ba447c968f833ed3b21d2a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0813ec8990ba447c968f833ed3b21d2a 2024-12-07T18:16:24,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0813ec8990ba447c968f833ed3b21d2a, entries=150, sequenceid=491, filesize=12.0 K 2024-12-07T18:16:24,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ec33782b2c22c62c1e44260bfbcf6877 in 860ms, sequenceid=491, compaction requested=true 2024-12-07T18:16:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:16:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:24,543 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:16:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:16:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:16:24,543 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:16:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:24,544 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:16:24,544 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133906 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:16:24,544 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/B is initiating minor compaction (all files) 2024-12-07T18:16:24,544 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/A is initiating minor compaction (all files) 2024-12-07T18:16:24,544 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/B in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:24,544 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/A in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:24,544 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/c7221637dab6487a8d577fbf7b17d630, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/8bdd4211bd9040abbfffd2b9ad4d6396, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/620ea3488df8445bbdf6fd4ca9b73b95, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/93fbe0d46f844defbace3e98cc88ffa0] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=48.2 K 2024-12-07T18:16:24,544 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/49b69bec64f64b5c93b8275d93288e6e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/3666ab865fa74fb682f0ff3f0eb12315, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/a8d4f3f1ae404b31ba1b9c0932c4b442, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/20c19b2376dc4196b3b3165ec36c0bc2] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=130.8 K 2024-12-07T18:16:24,544 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:24,544 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/49b69bec64f64b5c93b8275d93288e6e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/3666ab865fa74fb682f0ff3f0eb12315, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/a8d4f3f1ae404b31ba1b9c0932c4b442, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/20c19b2376dc4196b3b3165ec36c0bc2] 2024-12-07T18:16:24,544 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting c7221637dab6487a8d577fbf7b17d630, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733595380371 2024-12-07T18:16:24,545 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49b69bec64f64b5c93b8275d93288e6e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733595380371 2024-12-07T18:16:24,545 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bdd4211bd9040abbfffd2b9ad4d6396, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733595381224 2024-12-07T18:16:24,545 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3666ab865fa74fb682f0ff3f0eb12315, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733595381224 2024-12-07T18:16:24,545 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 620ea3488df8445bbdf6fd4ca9b73b95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733595381357 2024-12-07T18:16:24,546 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d4f3f1ae404b31ba1b9c0932c4b442, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733595381357 2024-12-07T18:16:24,546 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 93fbe0d46f844defbace3e98cc88ffa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733595383681 2024-12-07T18:16:24,546 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20c19b2376dc4196b3b3165ec36c0bc2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733595381507 2024-12-07T18:16:24,554 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:24,555 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#B#compaction#510 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:24,555 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/76034dad3ce343afaa6851c7822a29bb is 50, key is test_row_0/B:col10/1733595383681/Put/seqid=0 2024-12-07T18:16:24,556 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412073cef69ccff864976b1557704224adba2_ec33782b2c22c62c1e44260bfbcf6877 store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:24,559 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412073cef69ccff864976b1557704224adba2_ec33782b2c22c62c1e44260bfbcf6877, store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:24,559 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412073cef69ccff864976b1557704224adba2_ec33782b2c22c62c1e44260bfbcf6877 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:24,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742431_1607 (size=12779) 2024-12-07T18:16:24,566 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/76034dad3ce343afaa6851c7822a29bb as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/76034dad3ce343afaa6851c7822a29bb 2024-12-07T18:16:24,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742432_1608 (size=4469) 2024-12-07T18:16:24,571 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#A#compaction#511 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:24,572 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/B of ec33782b2c22c62c1e44260bfbcf6877 into 76034dad3ce343afaa6851c7822a29bb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:24,572 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:24,572 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/cc7310293b1b4b909cf982ff0d43c0b6 is 175, key is test_row_0/A:col10/1733595383681/Put/seqid=0 2024-12-07T18:16:24,572 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/B, priority=12, startTime=1733595384543; duration=0sec 2024-12-07T18:16:24,572 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:24,572 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:B 2024-12-07T18:16:24,572 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-07T18:16:24,573 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-07T18:16:24,573 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/C is initiating minor compaction (all files) 2024-12-07T18:16:24,574 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/C in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:24,574 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d4df26893bb449a094e75b92e739ea50, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/12b58fe369ac4ff2839a8eb4173ac7ba, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/7b71aaeb6e734b14b2ed7afb54b6db9b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0813ec8990ba447c968f833ed3b21d2a] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=48.2 K 2024-12-07T18:16:24,574 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting d4df26893bb449a094e75b92e739ea50, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1733595380371 2024-12-07T18:16:24,574 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 12b58fe369ac4ff2839a8eb4173ac7ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1733595381224 2024-12-07T18:16:24,575 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b71aaeb6e734b14b2ed7afb54b6db9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1733595381357 2024-12-07T18:16:24,575 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 0813ec8990ba447c968f833ed3b21d2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733595383681 2024-12-07T18:16:24,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742433_1609 (size=31733) 2024-12-07T18:16:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-07T18:16:24,578 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:24,579 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:16:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-07T18:16:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T18:16:24,581 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:16:24,582 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:16:24,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:16:24,585 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/cc7310293b1b4b909cf982ff0d43c0b6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/cc7310293b1b4b909cf982ff0d43c0b6 2024-12-07T18:16:24,594 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/A of ec33782b2c22c62c1e44260bfbcf6877 into cc7310293b1b4b909cf982ff0d43c0b6(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:24,594 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:24,595 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/A, priority=12, startTime=1733595384543; duration=0sec 2024-12-07T18:16:24,595 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:24,595 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:A 2024-12-07T18:16:24,595 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#C#compaction#512 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:24,596 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/afb4b28ae9b142008375cf8a3c338df1 is 50, key is test_row_0/C:col10/1733595383681/Put/seqid=0 2024-12-07T18:16:24,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742434_1610 (size=12779) 2024-12-07T18:16:24,608 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/afb4b28ae9b142008375cf8a3c338df1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/afb4b28ae9b142008375cf8a3c338df1 2024-12-07T18:16:24,612 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/C of ec33782b2c22c62c1e44260bfbcf6877 into afb4b28ae9b142008375cf8a3c338df1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:24,612 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:24,612 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/C, priority=12, startTime=1733595384543; duration=0sec 2024-12-07T18:16:24,612 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:24,613 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:C 2024-12-07T18:16:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T18:16:24,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-07T18:16:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:24,735 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T18:16:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:24,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207feb3fa47e42446c6a8196c9f5d5a06f6_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595383706/Put/seqid=0 2024-12-07T18:16:24,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742435_1611 (size=12454) 2024-12-07T18:16:24,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:24,749 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207feb3fa47e42446c6a8196c9f5d5a06f6_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207feb3fa47e42446c6a8196c9f5d5a06f6_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/6ea56cf9886f40d9a98ac82bca4579dd, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:24,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/6ea56cf9886f40d9a98ac82bca4579dd is 175, key is test_row_0/A:col10/1733595383706/Put/seqid=0 2024-12-07T18:16:24,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742436_1612 (size=31255) 2024-12-07T18:16:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:24,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595444874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595444875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,879 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,880 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595444877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,883 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T18:16:24,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595444988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595444988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595444988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:25,154 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=512, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/6ea56cf9886f40d9a98ac82bca4579dd 2024-12-07T18:16:25,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/7e699b89cf8f4a628afeba3d2eb503af is 50, key is test_row_0/B:col10/1733595383706/Put/seqid=0 2024-12-07T18:16:25,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742437_1613 (size=12301) 2024-12-07T18:16:25,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595445198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:25,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:25,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595445198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:25,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595445198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T18:16:25,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595445508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:25,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595445508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:25,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595445508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:25,565 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/7e699b89cf8f4a628afeba3d2eb503af 2024-12-07T18:16:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/981854238d06481ea113a900f1c545c3 is 50, key is test_row_0/C:col10/1733595383706/Put/seqid=0 2024-12-07T18:16:25,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742438_1614 (size=12301) 2024-12-07T18:16:25,575 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=512 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/981854238d06481ea113a900f1c545c3 2024-12-07T18:16:25,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/6ea56cf9886f40d9a98ac82bca4579dd as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/6ea56cf9886f40d9a98ac82bca4579dd 2024-12-07T18:16:25,581 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/6ea56cf9886f40d9a98ac82bca4579dd, entries=150, sequenceid=512, filesize=30.5 K 2024-12-07T18:16:25,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/7e699b89cf8f4a628afeba3d2eb503af as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7e699b89cf8f4a628afeba3d2eb503af 2024-12-07T18:16:25,585 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7e699b89cf8f4a628afeba3d2eb503af, entries=150, sequenceid=512, filesize=12.0 K 2024-12-07T18:16:25,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/981854238d06481ea113a900f1c545c3 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/981854238d06481ea113a900f1c545c3 2024-12-07T18:16:25,588 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/981854238d06481ea113a900f1c545c3, entries=150, sequenceid=512, filesize=12.0 K 2024-12-07T18:16:25,589 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ec33782b2c22c62c1e44260bfbcf6877 in 854ms, sequenceid=512, compaction requested=false 2024-12-07T18:16:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-07T18:16:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-07T18:16:25,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-07T18:16:25,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0090 sec 2024-12-07T18:16:25,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.0140 sec 2024-12-07T18:16:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-07T18:16:25,719 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:25,719 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:16:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-07T18:16:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T18:16:25,721 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:16:25,722 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:16:25,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:16:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T18:16:25,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-07T18:16:25,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:25,874 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-07T18:16:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:25,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:25,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120740f5a855d2a044c991236e2577919387_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595384871/Put/seqid=0 2024-12-07T18:16:25,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742439_1615 (size=12454) 2024-12-07T18:16:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:26,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T18:16:26,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595446038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595446042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595446043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595446148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595446158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595446158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:26,290 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120740f5a855d2a044c991236e2577919387_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120740f5a855d2a044c991236e2577919387_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:26,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/68c31342b76f4da7905172c4159602b9, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:26,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/68c31342b76f4da7905172c4159602b9 is 175, key is test_row_0/A:col10/1733595384871/Put/seqid=0 2024-12-07T18:16:26,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742440_1616 (size=31255) 2024-12-07T18:16:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T18:16:26,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595446358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595446368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595446368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595446668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595446678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595446678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:26,695 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=526, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/68c31342b76f4da7905172c4159602b9 2024-12-07T18:16:26,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/5ba02386d784417a816f834f2ec5905a is 50, key is test_row_0/B:col10/1733595384871/Put/seqid=0 2024-12-07T18:16:26,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742441_1617 (size=12301) 2024-12-07T18:16:26,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T18:16:27,107 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/5ba02386d784417a816f834f2ec5905a 2024-12-07T18:16:27,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/40256f3278e747ceb0e5a898d08c08f9 is 50, key is test_row_0/C:col10/1733595384871/Put/seqid=0 2024-12-07T18:16:27,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742442_1618 (size=12301) 2024-12-07T18:16:27,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595447178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:27,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595447188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595447188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:27,519 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=526 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/40256f3278e747ceb0e5a898d08c08f9 2024-12-07T18:16:27,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/68c31342b76f4da7905172c4159602b9 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/68c31342b76f4da7905172c4159602b9 2024-12-07T18:16:27,526 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/68c31342b76f4da7905172c4159602b9, entries=150, sequenceid=526, filesize=30.5 K 2024-12-07T18:16:27,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/5ba02386d784417a816f834f2ec5905a as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5ba02386d784417a816f834f2ec5905a 2024-12-07T18:16:27,529 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5ba02386d784417a816f834f2ec5905a, entries=150, sequenceid=526, filesize=12.0 K 2024-12-07T18:16:27,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/40256f3278e747ceb0e5a898d08c08f9 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/40256f3278e747ceb0e5a898d08c08f9 2024-12-07T18:16:27,533 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/40256f3278e747ceb0e5a898d08c08f9, entries=150, sequenceid=526, filesize=12.0 K 2024-12-07T18:16:27,534 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ec33782b2c22c62c1e44260bfbcf6877 in 1660ms, sequenceid=526, compaction requested=true 2024-12-07T18:16:27,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:27,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:27,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-07T18:16:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-07T18:16:27,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-07T18:16:27,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8130 sec 2024-12-07T18:16:27,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.8180 sec 2024-12-07T18:16:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-07T18:16:27,869 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:27,869 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:16:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-07T18:16:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T18:16:27,871 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:16:27,871 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:16:27,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:16:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T18:16:28,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-07T18:16:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:28,024 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T18:16:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:28,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:28,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e07ad4da33b14125bd88a38bfd16d2bd_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595386042/Put/seqid=0 2024-12-07T18:16:28,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742443_1619 (size=12454) 2024-12-07T18:16:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T18:16:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:28,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:28,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595448214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595448218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595448218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595448328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595448328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595448329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:28,437 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207e07ad4da33b14125bd88a38bfd16d2bd_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e07ad4da33b14125bd88a38bfd16d2bd_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:28,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/92efaf23c9384419bb03b6a69f01f44b, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/92efaf23c9384419bb03b6a69f01f44b is 175, key is test_row_0/A:col10/1733595386042/Put/seqid=0 2024-12-07T18:16:28,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742444_1620 (size=31255) 2024-12-07T18:16:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T18:16:28,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595448538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595448538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595448538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,541 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,843 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=544, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/92efaf23c9384419bb03b6a69f01f44b 2024-12-07T18:16:28,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/5da01a4674e7468e872a1b910a908be1 is 50, key is test_row_0/B:col10/1733595386042/Put/seqid=0 2024-12-07T18:16:28,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595448848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742445_1621 (size=12301) 2024-12-07T18:16:28,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595448848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595448848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:28,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T18:16:29,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46788 deadline: 1733595449088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:29,093 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_1' in ec33782b2c22c62c1e44260bfbcf6877 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:29,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:29,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:29,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:29,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46802 deadline: 1733595449118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:29,123 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 92acd7e2389b,33947,1733595217075 for 'test_row_2' in ec33782b2c22c62c1e44260bfbcf6877 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-07T18:16:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:29,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:29,254 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/5da01a4674e7468e872a1b910a908be1 2024-12-07T18:16:29,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/a48dba3008ef4b7b8b24ab2487b068fe is 50, key is test_row_0/C:col10/1733595386042/Put/seqid=0 2024-12-07T18:16:29,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742446_1622 (size=12301) 2024-12-07T18:16:29,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595449358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:29,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595449358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:29,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595449368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:29,369 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:29,664 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=544 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/a48dba3008ef4b7b8b24ab2487b068fe 2024-12-07T18:16:29,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/92efaf23c9384419bb03b6a69f01f44b as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/92efaf23c9384419bb03b6a69f01f44b 2024-12-07T18:16:29,671 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/92efaf23c9384419bb03b6a69f01f44b, entries=150, sequenceid=544, filesize=30.5 K 2024-12-07T18:16:29,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/5da01a4674e7468e872a1b910a908be1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5da01a4674e7468e872a1b910a908be1 2024-12-07T18:16:29,674 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5da01a4674e7468e872a1b910a908be1, entries=150, sequenceid=544, filesize=12.0 K 2024-12-07T18:16:29,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/a48dba3008ef4b7b8b24ab2487b068fe as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/a48dba3008ef4b7b8b24ab2487b068fe 2024-12-07T18:16:29,679 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/a48dba3008ef4b7b8b24ab2487b068fe, entries=150, sequenceid=544, filesize=12.0 K 2024-12-07T18:16:29,681 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for ec33782b2c22c62c1e44260bfbcf6877 in 1657ms, sequenceid=544, compaction requested=true 2024-12-07T18:16:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:29,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-07T18:16:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-07T18:16:29,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-07T18:16:29,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8120 sec 2024-12-07T18:16:29,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.8160 sec 2024-12-07T18:16:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-07T18:16:30,018 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:30,019 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-07T18:16:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-12-07T18:16:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T18:16:30,020 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-07T18:16:30,021 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-07T18:16:30,021 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-07T18:16:30,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T18:16:30,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33947 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-07T18:16:30,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:30,173 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-07T18:16:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:30,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:30,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412071ad6178771fa44aa8b54a15d7edf7902_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595388202/Put/seqid=0 2024-12-07T18:16:30,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742447_1623 (size=12454) 2024-12-07T18:16:30,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:30,188 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412071ad6178771fa44aa8b54a15d7edf7902_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071ad6178771fa44aa8b54a15d7edf7902_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:30,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/691ae212de2d407aa853ecf49eb317f4, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:30,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/691ae212de2d407aa853ecf49eb317f4 is 175, key is test_row_0/A:col10/1733595388202/Put/seqid=0 2024-12-07T18:16:30,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742448_1624 (size=31255) 2024-12-07T18:16:30,199 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=559, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/691ae212de2d407aa853ecf49eb317f4 2024-12-07T18:16:30,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/7d542ebf28aa4ed4944c00b922931061 is 50, key is test_row_0/B:col10/1733595388202/Put/seqid=0 2024-12-07T18:16:30,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742449_1625 (size=12301) 2024-12-07T18:16:30,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T18:16:30,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:30,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. as already flushing 2024-12-07T18:16:30,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595450404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:30,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595450408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:30,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595450410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:30,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595450518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:30,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595450518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:30,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595450518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:30,617 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/7d542ebf28aa4ed4944c00b922931061 2024-12-07T18:16:30,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/e29925958d2c425a9b3bb595de2e377e is 50, key is test_row_0/C:col10/1733595388202/Put/seqid=0 2024-12-07T18:16:30,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742450_1626 (size=12301) 2024-12-07T18:16:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T18:16:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595450728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:30,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:30,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595450728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:30,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595450728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:30,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:30,803 INFO [Thread-2457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2457. 2024-12-07T18:16:30,804 DEBUG [Thread-2457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:30,804 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:30,804 DEBUG [Thread-2457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:30,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:30,805 INFO [Thread-2465 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2465. 2024-12-07T18:16:30,805 DEBUG [Thread-2465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:30,805 DEBUG [Thread-2465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:30,805 DEBUG [Thread-2465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:30,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:30,808 INFO [Thread-2459 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2459. 2024-12-07T18:16:30,808 DEBUG [Thread-2459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:30,808 DEBUG [Thread-2459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:30,808 DEBUG [Thread-2459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:30,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:30,808 INFO [Thread-2463 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2463. 2024-12-07T18:16:30,808 DEBUG [Thread-2463 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:30,808 DEBUG [Thread-2463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:30,809 DEBUG [Thread-2463 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:30,809 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:30,809 INFO [Thread-2461 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2461. 2024-12-07T18:16:30,809 DEBUG [Thread-2461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:30,809 DEBUG [Thread-2461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:30,809 DEBUG [Thread-2461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:30,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:31,028 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/e29925958d2c425a9b3bb595de2e377e 2024-12-07T18:16:31,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/691ae212de2d407aa853ecf49eb317f4 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/691ae212de2d407aa853ecf49eb317f4 2024-12-07T18:16:31,033 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/691ae212de2d407aa853ecf49eb317f4, entries=150, sequenceid=559, filesize=30.5 K 2024-12-07T18:16:31,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/7d542ebf28aa4ed4944c00b922931061 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7d542ebf28aa4ed4944c00b922931061 2024-12-07T18:16:31,036 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7d542ebf28aa4ed4944c00b922931061, entries=150, sequenceid=559, filesize=12.0 K 2024-12-07T18:16:31,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/e29925958d2c425a9b3bb595de2e377e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e29925958d2c425a9b3bb595de2e377e 2024-12-07T18:16:31,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:31,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:31,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-07T18:16:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46846 deadline: 1733595451038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46818 deadline: 1733595451038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33947 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46862 deadline: 1733595451038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 , the old value is region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-07T18:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:31,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., hostname=92acd7e2389b,33947,1733595217075, seqNum=5 because the exception is null or not the one we care about 2024-12-07T18:16:31,039 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e29925958d2c425a9b3bb595de2e377e, entries=150, sequenceid=559, filesize=12.0 K 2024-12-07T18:16:31,039 INFO [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ec33782b2c22c62c1e44260bfbcf6877 in 866ms, sequenceid=559, compaction requested=true 2024-12-07T18:16:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:31,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/92acd7e2389b:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-12-07T18:16:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-12-07T18:16:31,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-07T18:16:31,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0200 sec 2024-12-07T18:16:31,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.0230 sec 2024-12-07T18:16:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-07T18:16:31,158 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:31,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33947 {}] regionserver.HRegion(8855): Flush requested on ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:31,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-07T18:16:31,550 INFO [Thread-2454 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2454. 2024-12-07T18:16:31,550 DEBUG [Thread-2454 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:31,550 DEBUG [Thread-2454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:31,551 DEBUG [Thread-2454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:31,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:31,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:31,551 INFO [Thread-2450 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2450. 2024-12-07T18:16:31,551 DEBUG [Thread-2450 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:31,551 DEBUG [Thread-2450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:31,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:31,551 INFO [Thread-2446 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2446. 2024-12-07T18:16:31,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:31,551 DEBUG [Thread-2446 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:31,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:31,551 DEBUG [Thread-2450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:31,551 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:31,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:31,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:31,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:31,551 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:31,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:31,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120713a49635a80b413d90c5a10db6062e17_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_0/A:col10/1733595390409/Put/seqid=0 2024-12-07T18:16:31,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742451_1627 (size=12454) 2024-12-07T18:16:31,959 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:31,962 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120713a49635a80b413d90c5a10db6062e17_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120713a49635a80b413d90c5a10db6062e17_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:31,962 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/0a98867c1b1d4b539747f4f8eb06afb6, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:31,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/0a98867c1b1d4b539747f4f8eb06afb6 is 175, key is test_row_0/A:col10/1733595390409/Put/seqid=0 2024-12-07T18:16:31,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742452_1628 (size=31255) 2024-12-07T18:16:32,366 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=579, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/0a98867c1b1d4b539747f4f8eb06afb6 2024-12-07T18:16:32,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/cceaea0d034d4243b0994688f06900ba is 50, key is test_row_0/B:col10/1733595390409/Put/seqid=0 2024-12-07T18:16:32,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742453_1629 (size=12301) 2024-12-07T18:16:32,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/cceaea0d034d4243b0994688f06900ba 2024-12-07T18:16:32,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/e6161da892c04713aeabc894bfa6d721 is 50, key is test_row_0/C:col10/1733595390409/Put/seqid=0 2024-12-07T18:16:32,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742454_1630 (size=12301) 2024-12-07T18:16:33,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/e6161da892c04713aeabc894bfa6d721 2024-12-07T18:16:33,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/0a98867c1b1d4b539747f4f8eb06afb6 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/0a98867c1b1d4b539747f4f8eb06afb6 2024-12-07T18:16:33,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/0a98867c1b1d4b539747f4f8eb06afb6, entries=150, sequenceid=579, filesize=30.5 K 2024-12-07T18:16:33,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/cceaea0d034d4243b0994688f06900ba as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/cceaea0d034d4243b0994688f06900ba 2024-12-07T18:16:33,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/cceaea0d034d4243b0994688f06900ba, entries=150, sequenceid=579, filesize=12.0 K 2024-12-07T18:16:33,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/e6161da892c04713aeabc894bfa6d721 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e6161da892c04713aeabc894bfa6d721 2024-12-07T18:16:33,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e6161da892c04713aeabc894bfa6d721, entries=150, sequenceid=579, filesize=12.0 K 2024-12-07T18:16:33,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for ec33782b2c22c62c1e44260bfbcf6877 in 1645ms, sequenceid=579, compaction requested=true 2024-12-07T18:16:33,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:33,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:A, priority=-2147483648, current under compaction store size is 1 2024-12-07T18:16:33,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:33,195 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-07T18:16:33,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:B, priority=-2147483648, current under compaction store size is 2 2024-12-07T18:16:33,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:33,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ec33782b2c22c62c1e44260bfbcf6877:C, priority=-2147483648, current under compaction store size is 3 2024-12-07T18:16:33,195 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-07T18:16:33,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:33,196 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 188008 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-07T18:16:33,196 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74284 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-07T18:16:33,197 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/A is initiating minor compaction (all files) 2024-12-07T18:16:33,197 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/B is initiating minor compaction (all files) 2024-12-07T18:16:33,197 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/A in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:33,197 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/B in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:33,197 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/cc7310293b1b4b909cf982ff0d43c0b6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/6ea56cf9886f40d9a98ac82bca4579dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/68c31342b76f4da7905172c4159602b9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/92efaf23c9384419bb03b6a69f01f44b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/691ae212de2d407aa853ecf49eb317f4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/0a98867c1b1d4b539747f4f8eb06afb6] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=183.6 K 2024-12-07T18:16:33,197 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/76034dad3ce343afaa6851c7822a29bb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7e699b89cf8f4a628afeba3d2eb503af, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5ba02386d784417a816f834f2ec5905a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5da01a4674e7468e872a1b910a908be1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7d542ebf28aa4ed4944c00b922931061, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/cceaea0d034d4243b0994688f06900ba] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=72.5 K 2024-12-07T18:16:33,197 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:33,197 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. files: [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/cc7310293b1b4b909cf982ff0d43c0b6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/6ea56cf9886f40d9a98ac82bca4579dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/68c31342b76f4da7905172c4159602b9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/92efaf23c9384419bb03b6a69f01f44b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/691ae212de2d407aa853ecf49eb317f4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/0a98867c1b1d4b539747f4f8eb06afb6] 2024-12-07T18:16:33,197 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc7310293b1b4b909cf982ff0d43c0b6, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733595383681 2024-12-07T18:16:33,197 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 76034dad3ce343afaa6851c7822a29bb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733595383681 2024-12-07T18:16:33,197 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea56cf9886f40d9a98ac82bca4579dd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733595383706 2024-12-07T18:16:33,197 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e699b89cf8f4a628afeba3d2eb503af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733595383706 2024-12-07T18:16:33,197 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ba02386d784417a816f834f2ec5905a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733595384871 2024-12-07T18:16:33,197 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68c31342b76f4da7905172c4159602b9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733595384871 2024-12-07T18:16:33,198 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 5da01a4674e7468e872a1b910a908be1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733595386029 2024-12-07T18:16:33,198 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92efaf23c9384419bb03b6a69f01f44b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733595386029 2024-12-07T18:16:33,198 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d542ebf28aa4ed4944c00b922931061, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1733595388202 2024-12-07T18:16:33,198 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 691ae212de2d407aa853ecf49eb317f4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1733595388202 2024-12-07T18:16:33,198 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting cceaea0d034d4243b0994688f06900ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1733595390407 2024-12-07T18:16:33,198 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a98867c1b1d4b539747f4f8eb06afb6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1733595390407 2024-12-07T18:16:33,205 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:33,207 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#B#compaction#528 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:33,207 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/c7a06e9c59794901bac75e18f99e1d2f is 50, key is test_row_0/B:col10/1733595390409/Put/seqid=0 2024-12-07T18:16:33,208 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241207017d56790e0d481e8fb91747cf49333e_ec33782b2c22c62c1e44260bfbcf6877 store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:33,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742455_1631 (size=12983) 2024-12-07T18:16:33,215 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/c7a06e9c59794901bac75e18f99e1d2f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/c7a06e9c59794901bac75e18f99e1d2f 2024-12-07T18:16:33,220 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/B of ec33782b2c22c62c1e44260bfbcf6877 into c7a06e9c59794901bac75e18f99e1d2f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:33,220 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:33,220 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/B, priority=10, startTime=1733595393195; duration=0sec 2024-12-07T18:16:33,220 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-07T18:16:33,220 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:B 2024-12-07T18:16:33,220 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-12-07T18:16:33,222 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 74284 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-12-07T18:16:33,222 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1541): ec33782b2c22c62c1e44260bfbcf6877/C is initiating minor compaction (all files) 2024-12-07T18:16:33,222 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ec33782b2c22c62c1e44260bfbcf6877/C in TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:33,222 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/afb4b28ae9b142008375cf8a3c338df1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/981854238d06481ea113a900f1c545c3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/40256f3278e747ceb0e5a898d08c08f9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/a48dba3008ef4b7b8b24ab2487b068fe, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e29925958d2c425a9b3bb595de2e377e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e6161da892c04713aeabc894bfa6d721] into tmpdir=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp, totalSize=72.5 K 2024-12-07T18:16:33,222 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting afb4b28ae9b142008375cf8a3c338df1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1733595383681 2024-12-07T18:16:33,223 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 981854238d06481ea113a900f1c545c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=512, earliestPutTs=1733595383706 2024-12-07T18:16:33,223 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting 40256f3278e747ceb0e5a898d08c08f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=526, earliestPutTs=1733595384871 2024-12-07T18:16:33,223 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting a48dba3008ef4b7b8b24ab2487b068fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=544, earliestPutTs=1733595386029 2024-12-07T18:16:33,223 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e29925958d2c425a9b3bb595de2e377e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1733595388202 2024-12-07T18:16:33,224 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] compactions.Compactor(225): Compacting e6161da892c04713aeabc894bfa6d721, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1733595390407 2024-12-07T18:16:33,224 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241207017d56790e0d481e8fb91747cf49333e_ec33782b2c22c62c1e44260bfbcf6877, store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:33,225 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241207017d56790e0d481e8fb91747cf49333e_ec33782b2c22c62c1e44260bfbcf6877 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:33,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742456_1632 (size=4469) 2024-12-07T18:16:33,234 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#A#compaction#529 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:33,235 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/18bcd2f8c0464e15a8f3caecade9d94e is 175, key is test_row_0/A:col10/1733595390409/Put/seqid=0 2024-12-07T18:16:33,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742457_1633 (size=31937) 2024-12-07T18:16:33,248 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ec33782b2c22c62c1e44260bfbcf6877#C#compaction#530 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-07T18:16:33,249 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/244bdff7baa14e49b016ce846fc01a36 is 50, key is test_row_0/C:col10/1733595390409/Put/seqid=0 2024-12-07T18:16:33,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742458_1634 (size=12983) 2024-12-07T18:16:33,649 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/18bcd2f8c0464e15a8f3caecade9d94e as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/18bcd2f8c0464e15a8f3caecade9d94e 2024-12-07T18:16:33,652 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/A of ec33782b2c22c62c1e44260bfbcf6877 into 18bcd2f8c0464e15a8f3caecade9d94e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:33,652 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:33,652 INFO [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/A, priority=10, startTime=1733595393195; duration=0sec 2024-12-07T18:16:33,652 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:33,652 DEBUG [RS:0;92acd7e2389b:33947-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:A 2024-12-07T18:16:33,655 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/244bdff7baa14e49b016ce846fc01a36 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/244bdff7baa14e49b016ce846fc01a36 2024-12-07T18:16:33,658 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in ec33782b2c22c62c1e44260bfbcf6877/C of ec33782b2c22c62c1e44260bfbcf6877 into 244bdff7baa14e49b016ce846fc01a36(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-07T18:16:33,658 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ec33782b2c22c62c1e44260bfbcf6877: 2024-12-07T18:16:33,658 INFO [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., storeName=ec33782b2c22c62c1e44260bfbcf6877/C, priority=10, startTime=1733595393195; duration=0sec 2024-12-07T18:16:33,658 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-07T18:16:33,659 DEBUG [RS:0;92acd7e2389b:33947-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ec33782b2c22c62c1e44260bfbcf6877:C 2024-12-07T18:16:35,435 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-07T18:16:39,110 INFO [Thread-2452 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2452. 2024-12-07T18:16:39,110 DEBUG [Thread-2452 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:39,110 DEBUG [Thread-2452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:39,110 DEBUG [Thread-2452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:39,110 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:39,179 INFO [Thread-2448 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2448. 2024-12-07T18:16:39,179 DEBUG [Thread-2448 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-07T18:16:39,179 DEBUG [Thread-2448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:39,179 DEBUG [Thread-2448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:39,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:39,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 167 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 169 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3044 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3047 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3034 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3048 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3051 2024-12-07T18:16:39,180 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-07T18:16:39,180 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:16:39,180 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:16:39,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:39,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:39,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:39,181 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-07T18:16:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T18:16:39,183 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595399183"}]},"ts":"1733595399183"} 2024-12-07T18:16:39,184 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-07T18:16:39,184 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-07T18:16:39,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-07T18:16:39,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, UNASSIGN}] 2024-12-07T18:16:39,186 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, UNASSIGN 2024-12-07T18:16:39,187 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=ec33782b2c22c62c1e44260bfbcf6877, regionState=CLOSING, regionLocation=92acd7e2389b,33947,1733595217075 2024-12-07T18:16:39,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, UNASSIGN because future has completed 2024-12-07T18:16:39,188 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-07T18:16:39,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075}] 2024-12-07T18:16:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T18:16:39,340 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:39,340 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-07T18:16:39,340 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing ec33782b2c22c62c1e44260bfbcf6877, disabling compactions & flushes 2024-12-07T18:16:39,340 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:39,340 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:39,340 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. after waiting 0 ms 2024-12-07T18:16:39,340 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:39,340 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(2902): Flushing ec33782b2c22c62c1e44260bfbcf6877 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-12-07T18:16:39,341 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=A 2024-12-07T18:16:39,341 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:39,341 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=B 2024-12-07T18:16:39,341 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:39,341 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ec33782b2c22c62c1e44260bfbcf6877, store=C 2024-12-07T18:16:39,341 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-07T18:16:39,345 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120746a822e068724ffd9f5a46771460c5d7_ec33782b2c22c62c1e44260bfbcf6877 is 50, key is test_row_1/A:col10/1733595399109/Put/seqid=0 2024-12-07T18:16:39,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742459_1635 (size=9914) 2024-12-07T18:16:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T18:16:39,749 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T18:16:39,752 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120746a822e068724ffd9f5a46771460c5d7_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120746a822e068724ffd9f5a46771460c5d7_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:39,752 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/215719de97c84c558f6d8144eac8b4f8, store: [table=TestAcidGuarantees family=A region=ec33782b2c22c62c1e44260bfbcf6877] 2024-12-07T18:16:39,753 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/215719de97c84c558f6d8144eac8b4f8 is 175, key is test_row_1/A:col10/1733595399109/Put/seqid=0 2024-12-07T18:16:39,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742460_1636 (size=22561) 2024-12-07T18:16:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T18:16:40,156 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=587, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/215719de97c84c558f6d8144eac8b4f8 2024-12-07T18:16:40,161 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/d80d8bee90694620b6dca6dac8ce8e77 is 50, key is test_row_1/B:col10/1733595399109/Put/seqid=0 2024-12-07T18:16:40,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742461_1637 (size=9857) 2024-12-07T18:16:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T18:16:40,564 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/d80d8bee90694620b6dca6dac8ce8e77 2024-12-07T18:16:40,569 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/63450306f3b7405e9413a2817ac283dc is 50, key is test_row_1/C:col10/1733595399109/Put/seqid=0 2024-12-07T18:16:40,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742462_1638 (size=9857) 2024-12-07T18:16:40,973 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=587 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/63450306f3b7405e9413a2817ac283dc 2024-12-07T18:16:40,976 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/A/215719de97c84c558f6d8144eac8b4f8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/215719de97c84c558f6d8144eac8b4f8 2024-12-07T18:16:40,979 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/215719de97c84c558f6d8144eac8b4f8, entries=100, sequenceid=587, filesize=22.0 K 2024-12-07T18:16:40,979 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/B/d80d8bee90694620b6dca6dac8ce8e77 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d80d8bee90694620b6dca6dac8ce8e77 2024-12-07T18:16:40,982 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d80d8bee90694620b6dca6dac8ce8e77, entries=100, sequenceid=587, filesize=9.6 K 2024-12-07T18:16:40,982 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/.tmp/C/63450306f3b7405e9413a2817ac283dc as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/63450306f3b7405e9413a2817ac283dc 2024-12-07T18:16:40,985 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/63450306f3b7405e9413a2817ac283dc, entries=100, sequenceid=587, filesize=9.6 K 2024-12-07T18:16:40,986 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for ec33782b2c22c62c1e44260bfbcf6877 in 1645ms, sequenceid=587, compaction requested=false 2024-12-07T18:16:40,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/b431724f3e0f4b4fb39a2537de36c0f2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/2d94bf856b6b422093079756b4d4634e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/82bc91843a5c4467afab88a308016b9a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/34683531c8874f5caf394a223b35de14, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/f975014e4ccf4b3486133172d5b5d7e9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1dc00a78c5a64c30b7923aa3e723ef8b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/42e03efa5a5e40f9b010de6c07ffecb4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/fce1d0c097da4f3888c14016081a5025, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/452074764b13421ea53801fc574e8fa5, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1b32b9041ad642bfa530ef3f4c97fefa, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/49b69bec64f64b5c93b8275d93288e6e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/3666ab865fa74fb682f0ff3f0eb12315, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/a8d4f3f1ae404b31ba1b9c0932c4b442, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/20c19b2376dc4196b3b3165ec36c0bc2, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/cc7310293b1b4b909cf982ff0d43c0b6, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/6ea56cf9886f40d9a98ac82bca4579dd, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/68c31342b76f4da7905172c4159602b9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/92efaf23c9384419bb03b6a69f01f44b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/691ae212de2d407aa853ecf49eb317f4, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/0a98867c1b1d4b539747f4f8eb06afb6] to archive 2024-12-07T18:16:40,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:16:40,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/b431724f3e0f4b4fb39a2537de36c0f2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/b431724f3e0f4b4fb39a2537de36c0f2 2024-12-07T18:16:40,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/2d94bf856b6b422093079756b4d4634e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/2d94bf856b6b422093079756b4d4634e 2024-12-07T18:16:40,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/82bc91843a5c4467afab88a308016b9a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/82bc91843a5c4467afab88a308016b9a 2024-12-07T18:16:40,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/34683531c8874f5caf394a223b35de14 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/34683531c8874f5caf394a223b35de14 2024-12-07T18:16:40,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/f975014e4ccf4b3486133172d5b5d7e9 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/f975014e4ccf4b3486133172d5b5d7e9 2024-12-07T18:16:40,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1dc00a78c5a64c30b7923aa3e723ef8b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1dc00a78c5a64c30b7923aa3e723ef8b 2024-12-07T18:16:40,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/42e03efa5a5e40f9b010de6c07ffecb4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/42e03efa5a5e40f9b010de6c07ffecb4 2024-12-07T18:16:40,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/fce1d0c097da4f3888c14016081a5025 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/fce1d0c097da4f3888c14016081a5025 2024-12-07T18:16:40,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/452074764b13421ea53801fc574e8fa5 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/452074764b13421ea53801fc574e8fa5 2024-12-07T18:16:40,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1b32b9041ad642bfa530ef3f4c97fefa to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/1b32b9041ad642bfa530ef3f4c97fefa 2024-12-07T18:16:40,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/49b69bec64f64b5c93b8275d93288e6e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/49b69bec64f64b5c93b8275d93288e6e 2024-12-07T18:16:40,997 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/3666ab865fa74fb682f0ff3f0eb12315 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/3666ab865fa74fb682f0ff3f0eb12315 2024-12-07T18:16:40,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/a8d4f3f1ae404b31ba1b9c0932c4b442 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/a8d4f3f1ae404b31ba1b9c0932c4b442 2024-12-07T18:16:40,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/20c19b2376dc4196b3b3165ec36c0bc2 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/20c19b2376dc4196b3b3165ec36c0bc2 2024-12-07T18:16:40,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/cc7310293b1b4b909cf982ff0d43c0b6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/cc7310293b1b4b909cf982ff0d43c0b6 2024-12-07T18:16:41,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/6ea56cf9886f40d9a98ac82bca4579dd to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/6ea56cf9886f40d9a98ac82bca4579dd 2024-12-07T18:16:41,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/68c31342b76f4da7905172c4159602b9 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/68c31342b76f4da7905172c4159602b9 2024-12-07T18:16:41,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/92efaf23c9384419bb03b6a69f01f44b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/92efaf23c9384419bb03b6a69f01f44b 2024-12-07T18:16:41,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/691ae212de2d407aa853ecf49eb317f4 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/691ae212de2d407aa853ecf49eb317f4 2024-12-07T18:16:41,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/0a98867c1b1d4b539747f4f8eb06afb6 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/0a98867c1b1d4b539747f4f8eb06afb6 2024-12-07T18:16:41,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/54c683fdd3f546aa84fd9f8c249bae52, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/47b0bb6dd7524cffb336decf1e4a48b3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d92d9e6fcb0d4dcebfae813c29c81763, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1182147fc71e499191ff214fa8d06c99, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/290926a746e542f886fe84bb8502a3e3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d40a825114df49f6be3d075e1bdd4604, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1af3000bd4664018bca832f8dcf144b8, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/e9499df93b444b3ca740155825f91d4d, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/9110d01ee01d4ae288bf36532ff5a451, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/c7221637dab6487a8d577fbf7b17d630, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/11fc5c7355624a9fbb8e0a18bfcdb7df, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/8bdd4211bd9040abbfffd2b9ad4d6396, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/620ea3488df8445bbdf6fd4ca9b73b95, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/76034dad3ce343afaa6851c7822a29bb, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/93fbe0d46f844defbace3e98cc88ffa0, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7e699b89cf8f4a628afeba3d2eb503af, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5ba02386d784417a816f834f2ec5905a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5da01a4674e7468e872a1b910a908be1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7d542ebf28aa4ed4944c00b922931061, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/cceaea0d034d4243b0994688f06900ba] to archive 2024-12-07T18:16:41,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:16:41,005 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/54c683fdd3f546aa84fd9f8c249bae52 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/54c683fdd3f546aa84fd9f8c249bae52 2024-12-07T18:16:41,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/47b0bb6dd7524cffb336decf1e4a48b3 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/47b0bb6dd7524cffb336decf1e4a48b3 2024-12-07T18:16:41,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d92d9e6fcb0d4dcebfae813c29c81763 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d92d9e6fcb0d4dcebfae813c29c81763 2024-12-07T18:16:41,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1182147fc71e499191ff214fa8d06c99 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1182147fc71e499191ff214fa8d06c99 2024-12-07T18:16:41,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/290926a746e542f886fe84bb8502a3e3 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/290926a746e542f886fe84bb8502a3e3 2024-12-07T18:16:41,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d40a825114df49f6be3d075e1bdd4604 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d40a825114df49f6be3d075e1bdd4604 2024-12-07T18:16:41,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1af3000bd4664018bca832f8dcf144b8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/1af3000bd4664018bca832f8dcf144b8 2024-12-07T18:16:41,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/e9499df93b444b3ca740155825f91d4d to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/e9499df93b444b3ca740155825f91d4d 2024-12-07T18:16:41,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/9110d01ee01d4ae288bf36532ff5a451 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/9110d01ee01d4ae288bf36532ff5a451 2024-12-07T18:16:41,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/c7221637dab6487a8d577fbf7b17d630 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/c7221637dab6487a8d577fbf7b17d630 2024-12-07T18:16:41,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/11fc5c7355624a9fbb8e0a18bfcdb7df to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/11fc5c7355624a9fbb8e0a18bfcdb7df 2024-12-07T18:16:41,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/8bdd4211bd9040abbfffd2b9ad4d6396 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/8bdd4211bd9040abbfffd2b9ad4d6396 2024-12-07T18:16:41,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/620ea3488df8445bbdf6fd4ca9b73b95 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/620ea3488df8445bbdf6fd4ca9b73b95 2024-12-07T18:16:41,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/76034dad3ce343afaa6851c7822a29bb to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/76034dad3ce343afaa6851c7822a29bb 2024-12-07T18:16:41,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/93fbe0d46f844defbace3e98cc88ffa0 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/93fbe0d46f844defbace3e98cc88ffa0 2024-12-07T18:16:41,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7e699b89cf8f4a628afeba3d2eb503af to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7e699b89cf8f4a628afeba3d2eb503af 2024-12-07T18:16:41,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5ba02386d784417a816f834f2ec5905a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5ba02386d784417a816f834f2ec5905a 2024-12-07T18:16:41,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5da01a4674e7468e872a1b910a908be1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/5da01a4674e7468e872a1b910a908be1 2024-12-07T18:16:41,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7d542ebf28aa4ed4944c00b922931061 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/7d542ebf28aa4ed4944c00b922931061 2024-12-07T18:16:41,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/cceaea0d034d4243b0994688f06900ba to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/cceaea0d034d4243b0994688f06900ba 2024-12-07T18:16:41,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/94e04c56f9b749f39850420fe8399fcc, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/c56f306406ca4fcf952004a99076ef3c, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0b23a87149cb4971a72723ec297c3549, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/953499dccd7542b690b068352b000e41, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/3945541ed3d64e30843066aab2fc07a7, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/4c9ef0ecbe35462e81425bf84e89c8bf, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/18dbf76e353e4857a1835413aed50aee, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/9bf7010c6cb84383bdf9f302ba7bd95e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d0c494f81f224526befe48967e9e5322, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d4df26893bb449a094e75b92e739ea50, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/69799728d097445cb7afabb8da908c26, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/12b58fe369ac4ff2839a8eb4173ac7ba, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/7b71aaeb6e734b14b2ed7afb54b6db9b, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/afb4b28ae9b142008375cf8a3c338df1, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0813ec8990ba447c968f833ed3b21d2a, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/981854238d06481ea113a900f1c545c3, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/40256f3278e747ceb0e5a898d08c08f9, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/a48dba3008ef4b7b8b24ab2487b068fe, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e29925958d2c425a9b3bb595de2e377e, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e6161da892c04713aeabc894bfa6d721] to archive 2024-12-07T18:16:41,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-07T18:16:41,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/94e04c56f9b749f39850420fe8399fcc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/94e04c56f9b749f39850420fe8399fcc 2024-12-07T18:16:41,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/c56f306406ca4fcf952004a99076ef3c to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/c56f306406ca4fcf952004a99076ef3c 2024-12-07T18:16:41,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0b23a87149cb4971a72723ec297c3549 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0b23a87149cb4971a72723ec297c3549 2024-12-07T18:16:41,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/953499dccd7542b690b068352b000e41 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/953499dccd7542b690b068352b000e41 2024-12-07T18:16:41,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/3945541ed3d64e30843066aab2fc07a7 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/3945541ed3d64e30843066aab2fc07a7 2024-12-07T18:16:41,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/4c9ef0ecbe35462e81425bf84e89c8bf to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/4c9ef0ecbe35462e81425bf84e89c8bf 2024-12-07T18:16:41,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/18dbf76e353e4857a1835413aed50aee to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/18dbf76e353e4857a1835413aed50aee 2024-12-07T18:16:41,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/9bf7010c6cb84383bdf9f302ba7bd95e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/9bf7010c6cb84383bdf9f302ba7bd95e 2024-12-07T18:16:41,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d0c494f81f224526befe48967e9e5322 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d0c494f81f224526befe48967e9e5322 2024-12-07T18:16:41,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d4df26893bb449a094e75b92e739ea50 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/d4df26893bb449a094e75b92e739ea50 2024-12-07T18:16:41,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/69799728d097445cb7afabb8da908c26 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/69799728d097445cb7afabb8da908c26 2024-12-07T18:16:41,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/12b58fe369ac4ff2839a8eb4173ac7ba to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/12b58fe369ac4ff2839a8eb4173ac7ba 2024-12-07T18:16:41,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/7b71aaeb6e734b14b2ed7afb54b6db9b to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/7b71aaeb6e734b14b2ed7afb54b6db9b 2024-12-07T18:16:41,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/afb4b28ae9b142008375cf8a3c338df1 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/afb4b28ae9b142008375cf8a3c338df1 2024-12-07T18:16:41,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0813ec8990ba447c968f833ed3b21d2a to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/0813ec8990ba447c968f833ed3b21d2a 2024-12-07T18:16:41,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/981854238d06481ea113a900f1c545c3 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/981854238d06481ea113a900f1c545c3 2024-12-07T18:16:41,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/40256f3278e747ceb0e5a898d08c08f9 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/40256f3278e747ceb0e5a898d08c08f9 2024-12-07T18:16:41,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/a48dba3008ef4b7b8b24ab2487b068fe to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/a48dba3008ef4b7b8b24ab2487b068fe 2024-12-07T18:16:41,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e29925958d2c425a9b3bb595de2e377e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e29925958d2c425a9b3bb595de2e377e 2024-12-07T18:16:41,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e6161da892c04713aeabc894bfa6d721 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/e6161da892c04713aeabc894bfa6d721 2024-12-07T18:16:41,040 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/recovered.edits/590.seqid, newMaxSeqId=590, maxSeqId=4 2024-12-07T18:16:41,041 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. 2024-12-07T18:16:41,041 DEBUG [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for ec33782b2c22c62c1e44260bfbcf6877: Waiting for close lock at 1733595399340Running coprocessor pre-close hooks at 1733595399340Disabling compacts and flushes for region at 1733595399340Disabling writes for close at 1733595399340Obtaining lock to block concurrent updates at 1733595399340Preparing flush snapshotting stores in ec33782b2c22c62c1e44260bfbcf6877 at 1733595399340Finished memstore snapshotting TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1733595399341 (+1 ms)Flushing stores of TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877. at 1733595399341Flushing ec33782b2c22c62c1e44260bfbcf6877/A: creating writer at 1733595399341Flushing mob file ec33782b2c22c62c1e44260bfbcf6877/A: appending metadata at 1733595399345 (+4 ms)Flushing mob file ec33782b2c22c62c1e44260bfbcf6877/A: closing flushed file at 1733595399345Flushing ec33782b2c22c62c1e44260bfbcf6877/A: appending metadata at 1733595399752 (+407 ms)Flushing ec33782b2c22c62c1e44260bfbcf6877/A: closing flushed file at 1733595399753 (+1 ms)Flushing ec33782b2c22c62c1e44260bfbcf6877/B: creating writer at 1733595400159 (+406 ms)Flushing ec33782b2c22c62c1e44260bfbcf6877/B: appending metadata at 1733595400161 (+2 ms)Flushing ec33782b2c22c62c1e44260bfbcf6877/B: closing flushed file at 1733595400161Flushing ec33782b2c22c62c1e44260bfbcf6877/C: creating writer at 1733595400567 (+406 ms)Flushing ec33782b2c22c62c1e44260bfbcf6877/C: appending metadata at 1733595400569 (+2 ms)Flushing ec33782b2c22c62c1e44260bfbcf6877/C: closing flushed file at 1733595400569Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@aed8d32: reopening flushed file at 1733595400976 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73c67152: reopening flushed file at 1733595400979 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33d0a77d: reopening flushed file at 1733595400982 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for ec33782b2c22c62c1e44260bfbcf6877 in 1645ms, sequenceid=587, compaction requested=false at 1733595400986 (+4 ms)Writing region close event to WAL at 1733595401038 (+52 ms)Running coprocessor post-close hooks at 1733595401041 (+3 ms)Closed at 1733595401041 2024-12-07T18:16:41,042 INFO [RS_CLOSE_REGION-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,043 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=ec33782b2c22c62c1e44260bfbcf6877, regionState=CLOSED 2024-12-07T18:16:41,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 because future has completed 2024-12-07T18:16:41,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-12-07T18:16:41,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure ec33782b2c22c62c1e44260bfbcf6877, server=92acd7e2389b,33947,1733595217075 in 1.8560 sec 2024-12-07T18:16:41,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-12-07T18:16:41,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ec33782b2c22c62c1e44260bfbcf6877, UNASSIGN in 1.8600 sec 2024-12-07T18:16:41,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-07T18:16:41,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8620 sec 2024-12-07T18:16:41,049 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733595401049"}]},"ts":"1733595401049"} 2024-12-07T18:16:41,051 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-07T18:16:41,051 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-07T18:16:41,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8700 sec 2024-12-07T18:16:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-07T18:16:41,329 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:41,329 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-07T18:16:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:41,330 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-07T18:16:41,331 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:41,333 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,334 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C, FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/recovered.edits] 2024-12-07T18:16:41,336 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/18bcd2f8c0464e15a8f3caecade9d94e to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/18bcd2f8c0464e15a8f3caecade9d94e 2024-12-07T18:16:41,337 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/215719de97c84c558f6d8144eac8b4f8 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/A/215719de97c84c558f6d8144eac8b4f8 2024-12-07T18:16:41,338 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/c7a06e9c59794901bac75e18f99e1d2f to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/c7a06e9c59794901bac75e18f99e1d2f 2024-12-07T18:16:41,339 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d80d8bee90694620b6dca6dac8ce8e77 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/B/d80d8bee90694620b6dca6dac8ce8e77 2024-12-07T18:16:41,340 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/244bdff7baa14e49b016ce846fc01a36 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/244bdff7baa14e49b016ce846fc01a36 2024-12-07T18:16:41,341 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/63450306f3b7405e9413a2817ac283dc to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/C/63450306f3b7405e9413a2817ac283dc 2024-12-07T18:16:41,343 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/recovered.edits/590.seqid to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877/recovered.edits/590.seqid 2024-12-07T18:16:41,343 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/default/TestAcidGuarantees/ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,343 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-07T18:16:41,344 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T18:16:41,344 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-07T18:16:41,346 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120703e0c395a2c3498a885f5393e4fc8059_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120703e0c395a2c3498a885f5393e4fc8059_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,347 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120711fa6b37a99749fdab2b35578f1ce04c_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120711fa6b37a99749fdab2b35578f1ce04c_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,348 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120713a49635a80b413d90c5a10db6062e17_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120713a49635a80b413d90c5a10db6062e17_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,349 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071ad6178771fa44aa8b54a15d7edf7902_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071ad6178771fa44aa8b54a15d7edf7902_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,349 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071e304453ad374feaa1938fc67b2c782e_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412071e304453ad374feaa1938fc67b2c782e_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,350 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073c1396892be1401f94a22b2eaf131cc5_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412073c1396892be1401f94a22b2eaf131cc5_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,351 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120740f5a855d2a044c991236e2577919387_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120740f5a855d2a044c991236e2577919387_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,352 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120746a822e068724ffd9f5a46771460c5d7_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120746a822e068724ffd9f5a46771460c5d7_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,353 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207563a9695f9c940e48eba945a13f975be_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207563a9695f9c940e48eba945a13f975be_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,353 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207644066a5187a4d2c82cdde650bc4bf18_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207644066a5187a4d2c82cdde650bc4bf18_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,354 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412079b9db62428694f29a9d2e1b09e388442_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412079b9db62428694f29a9d2e1b09e388442_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,355 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a2b6a0688c124e72b87d8a8ea03aa1c3_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207a2b6a0688c124e72b87d8a8ea03aa1c3_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,356 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ac6f5a4d28284c88bb23360026c6b72c_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207ac6f5a4d28284c88bb23360026c6b72c_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,357 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207be1c5c747ac64014953c8dfd22c9aa26_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207be1c5c747ac64014953c8dfd22c9aa26_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,357 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c322abe5f37543a991a3f34fc915aabb_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207c322abe5f37543a991a3f34fc915aabb_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,358 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e07ad4da33b14125bd88a38bfd16d2bd_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207e07ad4da33b14125bd88a38bfd16d2bd_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,359 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207feb3fa47e42446c6a8196c9f5d5a06f6_ec33782b2c22c62c1e44260bfbcf6877 to hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241207feb3fa47e42446c6a8196c9f5d5a06f6_ec33782b2c22c62c1e44260bfbcf6877 2024-12-07T18:16:41,359 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-07T18:16:41,361 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:41,363 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-07T18:16:41,364 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-07T18:16:41,365 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:41,365 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-07T18:16:41,365 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733595401365"}]},"ts":"9223372036854775807"} 2024-12-07T18:16:41,367 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-07T18:16:41,367 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ec33782b2c22c62c1e44260bfbcf6877, NAME => 'TestAcidGuarantees,,1733595367707.ec33782b2c22c62c1e44260bfbcf6877.', STARTKEY => '', ENDKEY => ''}] 2024-12-07T18:16:41,367 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-07T18:16:41,367 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733595401367"}]},"ts":"9223372036854775807"} 2024-12-07T18:16:41,368 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-07T18:16:41,368 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-07T18:16:41,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-12-07T18:16:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38531 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-07T18:16:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-07T18:16:41,438 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-07T18:16:41,447 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 250), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 372), ProcessCount=11 (was 11), AvailableMemoryMB=5816 (was 5849) 2024-12-07T18:16:41,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T18:16:41,448 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:16:41,448 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:16:41,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:41,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:41,448 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T18:16:41,448 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T18:16:41,448 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1603076298, stopped=false 2024-12-07T18:16:41,448 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=92acd7e2389b,38531,1733595216342 2024-12-07T18:16:41,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T18:16:41,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T18:16:41,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:16:41,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:16:41,450 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T18:16:41,450 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T18:16:41,451 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:16:41,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:41,451 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '92acd7e2389b,33947,1733595217075' ***** 2024-12-07T18:16:41,451 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T18:16:41,451 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:16:41,451 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T18:16:41,451 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T18:16:41,452 INFO [RS:0;92acd7e2389b:33947 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T18:16:41,452 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T18:16:41,452 INFO [RS:0;92acd7e2389b:33947 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T18:16:41,452 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(959): stopping server 92acd7e2389b,33947,1733595217075 2024-12-07T18:16:41,452 INFO [RS:0;92acd7e2389b:33947 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T18:16:41,452 INFO [RS:0;92acd7e2389b:33947 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;92acd7e2389b:33947. 2024-12-07T18:16:41,452 DEBUG [RS:0;92acd7e2389b:33947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T18:16:41,452 DEBUG [RS:0;92acd7e2389b:33947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:41,452 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T18:16:41,453 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T18:16:41,453 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T18:16:41,453 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T18:16:41,453 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-07T18:16:41,453 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-07T18:16:41,453 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T18:16:41,454 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T18:16:41,454 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T18:16:41,454 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T18:16:41,454 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T18:16:41,454 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T18:16:41,454 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-07T18:16:41,477 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/info/f054883a4e084059bc44bb10e0a0b85f is 95, key is TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72./info:/1733595252548/DeleteFamily/seqid=0 2024-12-07T18:16:41,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742463_1639 (size=6514) 2024-12-07T18:16:41,499 INFO [regionserver/92acd7e2389b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:16:41,654 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T18:16:41,854 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T18:16:41,881 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/info/f054883a4e084059bc44bb10e0a0b85f 2024-12-07T18:16:41,899 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/ns/2bf6ff28379f4826b88647c08c8347d7 is 93, key is TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72./ns:/1733595252507/DeleteFamily/seqid=0 2024-12-07T18:16:41,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742464_1640 (size=5906) 2024-12-07T18:16:41,917 INFO [regionserver/92acd7e2389b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-07T18:16:41,917 INFO [regionserver/92acd7e2389b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-07T18:16:42,054 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T18:16:42,254 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T18:16:42,302 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/ns/2bf6ff28379f4826b88647c08c8347d7 2024-12-07T18:16:42,319 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/rep_barrier/02cf361b7c3a44ceba1f4105661955d8 is 102, key is TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72./rep_barrier:/1733595252507/DeleteFamily/seqid=0 2024-12-07T18:16:42,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742465_1641 (size=6025) 2024-12-07T18:16:42,454 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T18:16:42,655 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T18:16:42,723 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/rep_barrier/02cf361b7c3a44ceba1f4105661955d8 2024-12-07T18:16:42,741 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/table/4379016dd6cd4741aa6bcd21958eace1 is 96, key is TestAcidGuarantees,,1733595220068.66753c009d64518d3f4dfefd580afd72./table:/1733595252507/DeleteFamily/seqid=0 2024-12-07T18:16:42,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742466_1642 (size=5936) 2024-12-07T18:16:42,855 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T18:16:43,055 DEBUG [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-07T18:16:43,144 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/table/4379016dd6cd4741aa6bcd21958eace1 2024-12-07T18:16:43,148 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/info/f054883a4e084059bc44bb10e0a0b85f as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/info/f054883a4e084059bc44bb10e0a0b85f 2024-12-07T18:16:43,150 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/info/f054883a4e084059bc44bb10e0a0b85f, entries=12, sequenceid=90, filesize=6.4 K 2024-12-07T18:16:43,151 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/ns/2bf6ff28379f4826b88647c08c8347d7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/ns/2bf6ff28379f4826b88647c08c8347d7 2024-12-07T18:16:43,153 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/ns/2bf6ff28379f4826b88647c08c8347d7, entries=8, sequenceid=90, filesize=5.8 K 2024-12-07T18:16:43,154 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/rep_barrier/02cf361b7c3a44ceba1f4105661955d8 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/rep_barrier/02cf361b7c3a44ceba1f4105661955d8 2024-12-07T18:16:43,156 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/rep_barrier/02cf361b7c3a44ceba1f4105661955d8, entries=6, sequenceid=90, filesize=5.9 K 2024-12-07T18:16:43,156 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/.tmp/table/4379016dd6cd4741aa6bcd21958eace1 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/table/4379016dd6cd4741aa6bcd21958eace1 2024-12-07T18:16:43,158 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/table/4379016dd6cd4741aa6bcd21958eace1, entries=7, sequenceid=90, filesize=5.8 K 2024-12-07T18:16:43,159 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1705ms, sequenceid=90, compaction requested=false 2024-12-07T18:16:43,162 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-07T18:16:43,163 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T18:16:43,163 INFO [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T18:16:43,163 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733595401453Running coprocessor pre-close hooks at 1733595401453Disabling compacts and flushes for region at 1733595401453Disabling writes for close at 1733595401454 (+1 ms)Obtaining lock to block concurrent updates at 1733595401454Preparing flush snapshotting stores in 1588230740 at 1733595401454Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733595401454Flushing stores of hbase:meta,,1.1588230740 at 1733595401455 (+1 ms)Flushing 1588230740/info: creating writer at 1733595401455Flushing 1588230740/info: appending metadata at 1733595401477 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733595401477Flushing 1588230740/ns: creating writer at 1733595401884 (+407 ms)Flushing 1588230740/ns: appending metadata at 1733595401898 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733595401898Flushing 1588230740/rep_barrier: creating writer at 1733595402305 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733595402319 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733595402319Flushing 1588230740/table: creating writer at 1733595402726 (+407 ms)Flushing 1588230740/table: appending metadata at 1733595402740 (+14 ms)Flushing 1588230740/table: closing flushed file at 1733595402740Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@676d476b: reopening flushed file at 1733595403147 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37c63a4e: reopening flushed file at 1733595403151 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58c6d377: reopening flushed file at 1733595403153 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67fa73e6: reopening flushed file at 1733595403156 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1705ms, sequenceid=90, compaction requested=false at 1733595403159 (+3 ms)Writing region close event to WAL at 1733595403160 (+1 ms)Running coprocessor post-close hooks at 1733595403163 (+3 ms)Closed at 1733595403163 2024-12-07T18:16:43,163 DEBUG [RS_CLOSE_META-regionserver/92acd7e2389b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T18:16:43,255 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(976): stopping server 92acd7e2389b,33947,1733595217075; all regions closed. 2024-12-07T18:16:43,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741834_1010 (size=25344) 2024-12-07T18:16:43,262 DEBUG [RS:0;92acd7e2389b:33947 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/oldWALs 2024-12-07T18:16:43,262 INFO [RS:0;92acd7e2389b:33947 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 92acd7e2389b%2C33947%2C1733595217075.meta:.meta(num 1733595219627) 2024-12-07T18:16:43,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741833_1009 (size=18062850) 2024-12-07T18:16:43,265 DEBUG [RS:0;92acd7e2389b:33947 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/oldWALs 2024-12-07T18:16:43,265 INFO [RS:0;92acd7e2389b:33947 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 92acd7e2389b%2C33947%2C1733595217075:(num 1733595219075) 2024-12-07T18:16:43,265 DEBUG [RS:0;92acd7e2389b:33947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T18:16:43,265 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T18:16:43,266 INFO [RS:0;92acd7e2389b:33947 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T18:16:43,266 INFO [RS:0;92acd7e2389b:33947 {}] hbase.ChoreService(370): Chore service for: regionserver/92acd7e2389b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T18:16:43,266 INFO [RS:0;92acd7e2389b:33947 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T18:16:43,266 INFO [regionserver/92acd7e2389b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T18:16:43,267 INFO [RS:0;92acd7e2389b:33947 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33947 2024-12-07T18:16:43,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/92acd7e2389b,33947,1733595217075 2024-12-07T18:16:43,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T18:16:43,271 INFO [RS:0;92acd7e2389b:33947 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T18:16:43,276 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [92acd7e2389b,33947,1733595217075] 2024-12-07T18:16:43,278 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/92acd7e2389b,33947,1733595217075 already deleted, retry=false 2024-12-07T18:16:43,278 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 92acd7e2389b,33947,1733595217075 expired; onlineServers=0 2024-12-07T18:16:43,278 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '92acd7e2389b,38531,1733595216342' ***** 2024-12-07T18:16:43,278 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T18:16:43,278 INFO [M:0;92acd7e2389b:38531 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T18:16:43,278 INFO [M:0;92acd7e2389b:38531 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T18:16:43,278 DEBUG [M:0;92acd7e2389b:38531 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T18:16:43,279 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T18:16:43,279 DEBUG [M:0;92acd7e2389b:38531 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T18:16:43,279 DEBUG [master/92acd7e2389b:0:becomeActiveMaster-HFileCleaner.large.0-1733595218775 {}] cleaner.HFileCleaner(306): Exit Thread[master/92acd7e2389b:0:becomeActiveMaster-HFileCleaner.large.0-1733595218775,5,FailOnTimeoutGroup] 2024-12-07T18:16:43,279 DEBUG [master/92acd7e2389b:0:becomeActiveMaster-HFileCleaner.small.0-1733595218775 {}] cleaner.HFileCleaner(306): Exit Thread[master/92acd7e2389b:0:becomeActiveMaster-HFileCleaner.small.0-1733595218775,5,FailOnTimeoutGroup] 2024-12-07T18:16:43,279 INFO [M:0;92acd7e2389b:38531 {}] hbase.ChoreService(370): Chore service for: master/92acd7e2389b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T18:16:43,279 INFO [M:0;92acd7e2389b:38531 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T18:16:43,279 DEBUG [M:0;92acd7e2389b:38531 {}] master.HMaster(1795): Stopping service threads 2024-12-07T18:16:43,279 INFO [M:0;92acd7e2389b:38531 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T18:16:43,279 INFO [M:0;92acd7e2389b:38531 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T18:16:43,279 ERROR [M:0;92acd7e2389b:38531 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-07T18:16:43,280 INFO [M:0;92acd7e2389b:38531 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T18:16:43,280 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T18:16:43,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T18:16:43,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T18:16:43,281 DEBUG [M:0;92acd7e2389b:38531 {}] zookeeper.ZKUtil(347): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T18:16:43,281 WARN [M:0;92acd7e2389b:38531 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T18:16:43,281 INFO [M:0;92acd7e2389b:38531 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/.lastflushedseqids 2024-12-07T18:16:43,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742467_1643 (size=72) 2024-12-07T18:16:43,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:16:43,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33947-0x1018e21a11b0001, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:16:43,378 INFO [RS:0;92acd7e2389b:33947 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T18:16:43,378 INFO [RS:0;92acd7e2389b:33947 {}] regionserver.HRegionServer(1031): Exiting; stopping=92acd7e2389b,33947,1733595217075; zookeeper connection closed. 2024-12-07T18:16:43,378 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@692b88d6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@692b88d6 2024-12-07T18:16:43,379 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-07T18:16:43,691 INFO [M:0;92acd7e2389b:38531 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T18:16:43,691 INFO [M:0;92acd7e2389b:38531 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T18:16:43,692 DEBUG [M:0;92acd7e2389b:38531 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T18:16:43,692 INFO [M:0;92acd7e2389b:38531 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:16:43,692 DEBUG [M:0;92acd7e2389b:38531 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:16:43,692 DEBUG [M:0;92acd7e2389b:38531 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T18:16:43,692 DEBUG [M:0;92acd7e2389b:38531 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:16:43,692 INFO [M:0;92acd7e2389b:38531 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=799.12 KB heapSize=981.15 KB 2024-12-07T18:16:43,707 DEBUG [M:0;92acd7e2389b:38531 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fd6695cf604f4cb78df903b53f573ed7 is 82, key is hbase:meta,,1/info:regioninfo/1733595219711/Put/seqid=0 2024-12-07T18:16:43,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742468_1644 (size=5672) 2024-12-07T18:16:44,110 INFO [M:0;92acd7e2389b:38531 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2252 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fd6695cf604f4cb78df903b53f573ed7 2024-12-07T18:16:44,131 DEBUG [M:0;92acd7e2389b:38531 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/591cb1637d414b06add961c36146f574 is 2479, key is \x00\x00\x00\x00\x00\x00\x00!/proc:d/1733595255716/Put/seqid=0 2024-12-07T18:16:44,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742469_1645 (size=46045) 2024-12-07T18:16:44,534 INFO [M:0;92acd7e2389b:38531 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=798.57 KB at sequenceid=2252 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/591cb1637d414b06add961c36146f574 2024-12-07T18:16:44,537 INFO [M:0;92acd7e2389b:38531 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 591cb1637d414b06add961c36146f574 2024-12-07T18:16:44,552 DEBUG [M:0;92acd7e2389b:38531 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/020490fc7fa64d98954f8f760b21c4a7 is 69, key is 92acd7e2389b,33947,1733595217075/rs:state/1733595218845/Put/seqid=0 2024-12-07T18:16:44,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073742470_1646 (size=5156) 2024-12-07T18:16:44,955 INFO [M:0;92acd7e2389b:38531 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2252 (bloomFilter=true), to=hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/020490fc7fa64d98954f8f760b21c4a7 2024-12-07T18:16:44,959 DEBUG [M:0;92acd7e2389b:38531 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fd6695cf604f4cb78df903b53f573ed7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fd6695cf604f4cb78df903b53f573ed7 2024-12-07T18:16:44,961 INFO [M:0;92acd7e2389b:38531 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fd6695cf604f4cb78df903b53f573ed7, entries=8, sequenceid=2252, filesize=5.5 K 2024-12-07T18:16:44,962 DEBUG [M:0;92acd7e2389b:38531 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/591cb1637d414b06add961c36146f574 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/591cb1637d414b06add961c36146f574 2024-12-07T18:16:44,964 INFO [M:0;92acd7e2389b:38531 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 591cb1637d414b06add961c36146f574 2024-12-07T18:16:44,964 INFO [M:0;92acd7e2389b:38531 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/591cb1637d414b06add961c36146f574, entries=174, sequenceid=2252, filesize=45.0 K 2024-12-07T18:16:44,965 DEBUG [M:0;92acd7e2389b:38531 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/020490fc7fa64d98954f8f760b21c4a7 as hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/020490fc7fa64d98954f8f760b21c4a7 2024-12-07T18:16:44,967 INFO [M:0;92acd7e2389b:38531 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35585/user/jenkins/test-data/966d5019-b3f6-2eda-0637-270ea44c4b68/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/020490fc7fa64d98954f8f760b21c4a7, entries=1, sequenceid=2252, filesize=5.0 K 2024-12-07T18:16:44,968 INFO [M:0;92acd7e2389b:38531 {}] regionserver.HRegion(3140): Finished flush of dataSize ~799.12 KB/818303, heapSize ~980.85 KB/1004392, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2252, compaction requested=false 2024-12-07T18:16:44,969 INFO [M:0;92acd7e2389b:38531 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T18:16:44,969 DEBUG [M:0;92acd7e2389b:38531 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733595403692Disabling compacts and flushes for region at 1733595403692Disabling writes for close at 1733595403692Obtaining lock to block concurrent updates at 1733595403692Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733595403692Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=818303, getHeapSize=1004632, getOffHeapSize=0, getCellsCount=2451 at 1733595403692Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733595403693 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733595403693Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733595403707 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733595403707Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733595404114 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733595404130 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733595404130Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733595404538 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733595404552 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733595404552Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13688827: reopening flushed file at 1733595404958 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74015b09: reopening flushed file at 1733595404961 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d32fc14: reopening flushed file at 1733595404964 (+3 ms)Finished flush of dataSize ~799.12 KB/818303, heapSize ~980.85 KB/1004392, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2252, compaction requested=false at 1733595404968 (+4 ms)Writing region close event to WAL at 1733595404969 (+1 ms)Closed at 1733595404969 2024-12-07T18:16:44,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37437 is added to blk_1073741830_1006 (size=965898) 2024-12-07T18:16:44,972 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T18:16:44,972 INFO [M:0;92acd7e2389b:38531 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T18:16:44,972 INFO [M:0;92acd7e2389b:38531 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38531 2024-12-07T18:16:44,972 INFO [M:0;92acd7e2389b:38531 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T18:16:45,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:16:45,074 INFO [M:0;92acd7e2389b:38531 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T18:16:45,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38531-0x1018e21a11b0000, quorum=127.0.0.1:62924, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T18:16:45,079 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-07T18:16:45,081 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T18:16:45,081 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T18:16:45,082 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T18:16:45,082 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/hadoop.log.dir/,STOPPED} 2024-12-07T18:16:45,085 WARN [BP-419382850-172.17.0.2-1733595213355 heartbeating to localhost/127.0.0.1:35585 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-07T18:16:45,085 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-07T18:16:45,085 WARN [BP-419382850-172.17.0.2-1733595213355 heartbeating to localhost/127.0.0.1:35585 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-419382850-172.17.0.2-1733595213355 (Datanode Uuid cbc12a6e-c0e1-4814-8304-16012a73b004) service to localhost/127.0.0.1:35585 2024-12-07T18:16:45,085 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-07T18:16:45,087 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/cluster_452d438f-2bce-cc52-05da-ff1816a8b998/data/data1/current/BP-419382850-172.17.0.2-1733595213355 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T18:16:45,087 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/cluster_452d438f-2bce-cc52-05da-ff1816a8b998/data/data2/current/BP-419382850-172.17.0.2-1733595213355 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-07T18:16:45,088 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-07T18:16:45,095 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-07T18:16:45,095 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-07T18:16:45,095 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-07T18:16:45,095 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-07T18:16:45,095 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b66f679c-5baa-dba1-89ef-9c2b14822b98/hadoop.log.dir/,STOPPED} 2024-12-07T18:16:45,110 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T18:16:45,240 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down